Load <- function(var, exp = NULL, obs = NULL, sdates, nmember = NULL, 
                 nmemberobs = NULL, nleadtime = NULL, leadtimemin = 1, 
                 leadtimemax = NULL, storefreq = 'monthly', sampleperiod = 1, 
                 lonmin = 0, lonmax = 360, latmin = -90, latmax = 90, 
                 output = 'areave', method = 'conservative', grid = NULL, 
                 maskmod = vector("list", 15), maskobs = vector("list", 15), 
                 configfile = NULL, suffixexp = NULL, suffixobs = NULL, 
                 varmin = NULL, varmax = NULL, silent = FALSE, nprocs = NULL) {
  #library(parallel)
  #library(bigmemory)

  # Check parameters
  if (!(storefreq %in% c('monthly', 'daily'))) {
    stop("Error: parameter 'storefreq' is wrong, can take value 'daily' or 'monthly'.")
  }
  if (!(method %in% c('bilinear', 'bicubic', 'conservative', 'distance-weighted'))) {
    stop("Error: parameter 'method' is wrong, can take value 'bilinear', 'bicubic', 'conservative' or 'distance-weighted'.")
  }
  if (is.null(configfile)) {
    configfile <- system.file("config", "IC3.conf", package = "s2dverification")
  }
  if (!is.null(suffixexp)) {
    if (length(suffixexp) == 1) {
      suffixexp <- rep(suffixexp[1], length(exp))
    }
    if (length(suffixexp) != length(exp)) {
      stop("Error: parameter 'suffixexp' must contain 0, 1 or length(exp) values.")
    }
  }
  if (!is.null(suffixobs)) {
    if (length(suffixobs) == 1) {
      suffixobs <- rep(suffixobs[1], length(obs))
    }
    if (length(suffixobs) != length(obs)) {
      stop("Error: parameter 'suffixobs' must contain 0, 1 or length(obs) values.")
    }
  }
  if (!is.null(nmember) && !is.null(exp)) {
    if (length(nmember) == 1) {
      print(paste("Warning: 'nmember' should specify the number of members of each experimental dataset. Forcing to", nmember, "for all experiments."))
      nmember <- rep(nmember, length(exp))
    }
    if (length(nmember) != length(exp)) {
      stop("Error: 'nmember' should contain as many values as 'exp'.")
    } else if (any(is.na(nmember))) {
      nmember[which(is.na(nmember))] <- max(nmember, na.rm = TRUE)
    }
  }
  if (!is.null(nmemberobs) && !is.null(obs)) {
    if (length(nmemberobs) == 1) {
      print(paste("Warning: 'nmemberobs' should specify the number of members of each observational dataset. Forcing to", nmemberobs, "for all observations."))
      nmemberobs <- rep(nmemberobs, length(obs))
    }
    if (length(nmemberobs) != length(obs)) {
      stop("Error: 'nmemberobs' should contain as many values as 'obs'.")
    } else if (any(is.na(nmemberobs))) {
      nmemberobs[which(is.na(nmemberobs))] <- max(nmemberobs, na.rm = TRUE)
    }
  }
  remap <- switch(method, 'bilinear' = 'remapbil', 'bicubic' = 'remapbic', 
                  'conservative' = 'remapcon', 'distance-weighted' = 'remapdis')
  if (!(output %in% c('lonlat', 'lon', 'lat', 'areave'))) {
    stop(paste("Error: 'output' can only take values 'lonlat', 'lon', 'lat' or 'areave'."))
  }
  ## Make sure that longitude bounds are positive
  if (lonmin < 0) {
    lonmin <- lonmin + 360
  }
  if (lonmax < 0) {
    lonmax <- lonmax + 360
  }
<<<<<<< HEAD
  
  # 
  #  Parameters needed to load data
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # 
  ## The following variables contain the paths to the experimental datasets
  roota <- '/cfunas/exp/ENSEMBLES/decadal/MME/'
  rootb <- '/cfunas/exp/*/'
  rootc <- '/cfunas/exp/ENSEMBLES/'
  rootd <- '/cfunas/exp/CMIP5/decadal/'
  roote <- '/cfunas/exp/CMIP5/historical/'
  rootf <- '/cfunas/exp/ECMWF/seasonal/0001/s004/'
  rootg <- '/cfunas/exp/ECMWF/seasonal/0001/s003/m001/'
  rooth <- '/cfunas/exp/SPECS/decadal/'
  rooti <- '/cfunas/exp/NMME/'
  
  ## lstatms is a list of the possible atmospheric variables (2-dimensional).
  lstatms <- c('tas', 'prlr', 'tos', 'g500', 'g200', 'ta50', 'psl', 'hflsd',
               'hfssd', 'rls', 'rss', 'rsds', 'uas', 'vas')
  ## lstatm_stats is a list of the atmospheric statistical variables.               
  lstatm_stats <- c('tas_q90', 'tas_mean', 'tas_ndaysq90', 'prlr_q90', 
                    'prlr_mean', 'prlr_ndaysq90', 'tasmax_q90', 'tasmin_q10', 
                    'prlr_q90', 'prlr_q10','tasmax_nbdaysq90_1979-2005', 
                    'tasmin_nbdaysq10_1979-2005', 'prlr_nbdaysq90_1979-2005', 
                    'prlr_nbdaysq10_1979-2005','tasmax_nbdaysq90_1979-2012', 
                    'tasmin_nbdaysq10_1979-2012', 'prlr_nbdaysq90_1979-2012', 
                    'prlr_nbdaysq10_1979-2012','tasmax_nbdaysq90_1981-2010', 
                    'tasmin_nbdaysq10_1981-2010', 'prlr_nbdaysq90_1981-2010', 
                    'prlr_nbdaysq10_1981-2010')
  ## lstoces is a list of the possible oceanic variables (global means). Please, if
  ## you insert new variables, do not change the current order of the variables.
  lstoces <- c('siaN', 'sieN', 'sivN', 'siaS', 'sieS', 'sivS', 
               'sia_Amundsen_Sea', 'sie_Amundsen_Sea', 'siv_Amundsen_Sea',
               'sia_Antarctic_Atlantic_Sector', 'sie_Antarctic_Atlantic_Sector', 'siv_Antarctic_Atlantic_Sector',
               'sia_Antarctic_Indian_Sector', 'sie_Antarctic_Indian_Sector', 'siv_Antarctic_Indian_Sector',
               'sia_Antarctic_Ocean', 'sie_Antarctic_Ocean', 'siv_Antarctic_Ocean',
               'sia_ArctOcn', 'sie_ArctOcn', 'siv_ArctOcn',
               'sia_Arctic_Marginal_Seas', 'sie_Arctic_Marginal_Seas', 'siv_Arctic_Marginal_Seas',
               'sia_Arctic_Ocean', 'sie_Arctic_Ocean', 'siv_Arctic_Ocean',
               'sia_Arctic_Ocean_North_Atlantic', 'sie_Arctic_Ocean_North_Atlantic', 'siv_Arctic_Ocean_North_Atlantic',
               'sia_Atlantic_Ocean', 'sie_Atlantic_Ocean', 'siv_Atlantic_Ocean',
               'sia_Baffin', 'sie_Baffin', 'siv_Baffin',
               'sia_Baffin_Bay', 'sie_Baffin_Bay', 'siv_Baffin_Bay',
               'sia_Baltic_Sea', 'sie_Baltic_Sea', 'siv_Baltic_Sea',
               'sia_BarKara', 'sie_BarKara', 'siv_BarKara',
               'sia_Barents_Sea', 'sie_Barents_Sea', 'siv_Barents_Sea',
               'sia_Beaufort_Chukchi_Sea', 'sie_Beaufort_Chukchi_Sea', 'siv_Beaufort_Chukchi_Sea',
               'sia_Beaufort_Sea', 'sie_Beaufort_Sea', 'siv_Beaufort_Sea',
               'sia_Bellingshausen_Sea', 'sie_Bellingshausen_Sea', 'siv_Bellingshausen_Sea',
               'sia_Bering', 'sie_Bering', 'siv_Bering',
               'sia_Bering_Strait', 'sie_Bering_Strait', 'siv_Bering_Strait',
               'sia_CanArch', 'sie_CanArch', 'siv_CanArch',
               'sia_Canadian_Waters', 'sie_Canadian_Waters', 'siv_Canadian_Waters',
               'sia_Caspian_Sea', 'sie_Caspian_Sea', 'siv_Caspian_Sea',
               'sia_Central_Arctic', 'sie_Central_Arctic', 'siv_Central_Arctic',
               'sia_Chukchi_Sea', 'sie_Chukchi_Sea', 'siv_Chukchi_Sea',
               'sia_East_Siberian_Sea', 'sie_East_Siberian_Sea', 'siv_East_Siberian_Sea',
               'sia_Eastern_Central_Arctic', 'sie_Arctic_Marginal_Seas', 'siv_Arctic_Marginal_Seas',
               'sia_Fram_Strait', 'sie_Fram_Strait', 'siv_Fram_Strait',
               'sia_Global_Ocean', 'sie_Global_Ocean', 'siv_Global_Ocean',
               'sia_Greenland_Sea', 'sie_Greenland_Sea', 'siv_Greenland_Sea',
               'sia_Grnland', 'sie_Grnland', 'siv_Grnland',
               'sia_Hudson', 'sie_Hudson', 'siv_Hudson',
               'sia_Icelandic_Sea', 'sie_Icelandic_Sea', 'siv_Icelandic_Sea',
               'sia_Indian_Ocean', 'sie_Indian_Ocean', 'siv_Indian_Ocean',
               'sia_Indo_Pacific_Ocean', 'sie_Indo_Pacific_Ocean', 'siv_Indo_Pacific_Ocean',
               'sia_Kara_Gate_Strait', 'sie_Kara_Gate_Strait', 'siv_Kara_Gate_Strait',
               'sia_Kara_Sea', 'sie_Kara_Sea', 'siv_Kara_Sea',
               'sia_LAT80N', 'sie_LAT80N', 'siv_LAT80N',
               'sia_Labrador_Sea', 'sie_Labrador_Sea', 'siv_Labrador_Sea',
               'sia_Laptev_East_Siberian_Chukchi_Seas', 'sie_Laptev_East_Siberian_Chukchi_Seas', 'siv_Laptev_East_Siberian_Chukchi_Seas',
               'sia_Laptev_East_Siberian_Seas', 'sie_Laptev_East_Siberian_Seas', 'siv_Laptev_East_Siberian_Seas',
               'sia_Laptev_Sea', 'sie_Laptev_Sea', 'siv_Laptev_Sea',
               'sia_Lincoln_Sea', 'sie_Lincoln_Sea', 'siv_Lincoln_Sea',
               'sia_Mediterranean_Sea', 'sie_Mediterranean_Sea', 'siv_Mediterranean_Sea',
               'sia_Nares_Strait', 'sie_Nares_Strait', 'siv_Nares_Strait',
               'sia_Nordic_Barents_Seas', 'sie_Nordic_Barents_Seas', 'siv_Nordic_Barents_Seas',
               'sia_Nordic_Seas', 'sie_Nordic_Seas', 'siv_Nordic_Seas',
               'sia_NorthWest_Passage', 'sie_NorthWest_Passage', 'siv_NorthWest_Passage',
               'sia_North_Atlantic_Arctic', 'sie_North_Atlantic_Arctic', 'siv_North_Atlantic_Arctic',
               'sia_North_Atlantic_Ocean', 'sie_North_Atlantic_Ocean', 'siv_North_Atlantic_Ocean',
               'sia_North_Hemisphere_Ocean', 'sie_North_Hemisphere_Ocean', 'siv_North_Hemisphere_Ocean',
               'sia_Northern_Hemisphere', 'sie_Northern_Hemisphere', 'siv_Northern_Hemisphere',
               'sia_Norwegian_Sea', 'sie_Norwegian_Sea', 'siv_Norwegian_Sea',
               'sia_Okhotsk', 'sie_Okhotsk', 'siv_Okhotsk',
               'sia_OpenOcean', 'sie_OpenOcean', 'siv_OpenOcean',
               'sia_Pacific_Ocean', 'sie_Pacific_Ocean', 'siv_Pacific_Ocean',
               'sia_Ross_Sea', 'sie_Ross_Sea', 'siv_Ross_Sea',
               'sia_Serreze_Arctic', 'sie_Serreze_Arctic', 'siv_Serreze_Arctic',
               'sia_Southern_Hemisphere', 'sie_Southern_Hemisphere', 'siv_Southern_Hemisphere',
               'sia_StLawr', 'sie_StLawr', 'siv_StLawr',
               'sia_Subpolar_Gyre', 'sie_Subpolar_Gyre', 'siv_Subpolar_Gyre',
               'sia_TotalArc', 'sie_TotalArc', 'siv_TotalArc',
               'sia_Vilkitsky_Strait', 'sie_Vilkitsky_Strait', 'siv_Vilkitsky_Strait',
               'sia_Weddell_Sea', 'sie_Weddell_Sea', 'siv_Weddell_Sea',
               'sia_Western_Central_Arctic', 'sie_Western_Central_Arctic', 'siv_Western_Central_Arctic',
               'moc_40N55N_1-2km', 'moc_30N40N_1-2km', 
               'max_moc_38N50N_500m-2km', 'max_moc_40N', 
               'NAtl_stc_0N25N_0-200m', 'SAtl_stc_25S0S_0-200m', 
               'SPac_stc_25S0S_0-200m', 'NPac_stc_0N25N_0-200m', 'heatc', 
               '0-315_heatc', '373-657_heatc', '800-5350_heatc','mxl_heatc', 
               'NAtl_10N65N_heatc', 'NAtl_10N65N_0-315_heatc', 
               'NAtl_10N65N_373-657_heatc', 'NAtl_10N65N_800-5350_heatc', 
               'mxl_NAtl_10N65N_heatc', 'TAtl_30S30N_heatc', 
               'TAtl_30S30N_0-315_heatc', 'TAtl_30S30N_373-657_heatc', 
               'TAtl_30S30N_800-5350_heatc', 'mxl_TAtl_30S30N_heatc',
               'NPac_10N70N_heatc', 'NPac_10N70N_0-315_heatc', 
               'NPac_10N70N_373-657_heatc', 'NPac_10N70N_800-5350_heatc', 
               'mxl_NPac_10N70N_heatc', 'TPac_30S30N_heatc', 
               'TPac_30S30N_0-315_heatc', 'TPac_30S30N_373-657_heatc', 
               'TPac_30S30N_800-5350_heatc', 'mxl_TPac_30S30N_heatc', 
               'Arc_65N90N_heatc', 'Arc_65N90N_0-315_heatc', 
               'Arc_65N90N_373-657_heatc', 'Arc_65N90N_800-5350_heatc', 
               'mxl_Arc_65N90N_heatc', 'Ant_90S60S_heatc', 
               'Ant_90S60S_0-315_heatc', 'Ant_90S60S_373-657_heatc', 
               'Ant_90S60S_800-5350_heatc', 'mxl_Ant_90S60S_heatc', 
               'TInd_30S30N_heatc', 'TInd_30S30N_0-315_heatc', 
               'TInd_30S30N_373-657_heatc', 'TInd_30S30N_800-5350_heatc',
               'mxl_TInd_30S30N_heatc')
=======
>>>>>>> 00243c3acb14d45481699c234ef88919310f51ed

  # Extend the current environment with the functions and variables related to 
  # the configuration file mechanism which contain the information about the 
  # location of the datasets.
  # 'replace_values' is a named list that associates a variable name to an 
  # associated value. Initially it is filled with variables and values parsed
  # from the configuration file, but we can add or modify some values during
  # the execution to choose for example which start date we want to load.
  # When 'replaceVariablesInString' is called, all the variable accesses 
  # ($VARIABLE_NAME$) that appear in the string given as parameter are 
  # replaced by the associated value in 'replace_values'.
  #
  # LoadConfigurationFile will tell if the variable we are loading is 
  # 2-dimensional or not and if the datasets we are loading are stored in
  # file per member, file per ensemble or file per dataset format.
  putInDictionary <- replaceVariablesInString <- NULL
  replace_values <- is_2d_var <- exp_info <- obs_info <- NULL
  is_file_per_member_exp <- is_file_per_member_obs <- NULL
  is_file_per_dataset_exp <- is_file_per_dataset_obs <- NULL
  LoadConfigurationFile(environment())

  # Some extra checks now that we know the variable type
  if (!is_2d_var && (output != 'areave')) {
    print(paste("Warning: '", output, "' output format not allowed when loading global mean variables. Forcing to 'areave'.", 
                sep = ''))
    output <- 'areave'
  }

  # Initialize some variables that will take various values along the
  # execution
  latitudes <- longitudes <- NULL
  leadtimes <- NULL
  var_exp <- var_obs <- NULL

  # Start defining the dimensions of the output matrices
  nmod <- length(exp)
  nobs <- length(obs)
  nsdates <- length(sdates)

  # We will iterate over all the experiments, start dates and members and will open
  # the file pointed by the data in the configuration file.
  # If a file is found, we will open it and read its metadata to work out the 
  # remaining dimensions: members, leadtimes, longitudes and latitudes.
  #
  # At each iteration we will build a 'work piece' that will contain information
  # on the data we want to load from a file. For each file we will have one
  # work piece. These work pieces will be packages of information to be sent to
  # the various parallel processes. Each process will need this information to
  # access and manipulate the data according to the output type and other 
  # parameters. 
  if (!silent) {
    print("* Fetching first experimental files to work out 'var_exp' size...")
  }

  dataset_type <- 'exp'
  dim_exp <- NULL
  filename <- file_found <- tmp <- nltime <- NULL
  dims2define <- TRUE
  exp_work_pieces <- list()
  jmod <- 1
  while (jmod <= nmod) {
    replace_values <- putInDictionary(replace_values, "EXP_NAME", exp[jmod])
    dataset_info <- exp_info[[jmod]]
    replace_values <- putInDictionary(replace_values, "EXP_MAIN_PATH", dataset_info[1])
    replace_values <- putInDictionary(replace_values, "EXP_FILE_PATH", dataset_info[2])
    replace_values <- putInDictionary(replace_values, "GRID", dataset_info[3])
    replace_values <- putInDictionary(replace_values, "NC_VAR_NAME", dataset_info[4])
    namevar <- replaceVariablesInString(dataset_info[4], replace_values)
    if (is.null(suffixexp) || is.na(suffixexp[jmod])) {
      replace_values <- putInDictionary(replace_values, "SUFFIX", dataset_info[5])
    } else {
      replace_values <- putInDictionary(replace_values, "SUFFIX", suffixexp[jmod])
    }
    if (is.null(varmin)) {
      mod_var_min <- as.numeric(replaceVariablesInString(dataset_info[6], replace_values))
    } else {
      mod_var_min <- varmin
    }
    if (is.null(varmax)) {
      mod_var_max <- as.numeric(replaceVariablesInString(dataset_info[7], replace_values))
    } else {
      mod_var_max <- varmax
    }
    jsdate <- 1
    while (jsdate <= nsdates) {
      replace_values <- putInDictionary(replace_values, "START_DATE", sdates[jsdate])
      # If the dimensions of the output matrices are still to define, we try to read
      # the metadata of the data file that corresponds to the current iteration
      if (dims2define) {
        if (is_file_per_member_exp[jmod]) {
          replace_values <- putInDictionary(replace_values, "MEMBER_NUMBER", '*')
        }
        # We must build a work piece that will be sent to the LoadDataFile function
        # in 'explore_dims' mode. We will obtain, if success, the dimensions of the
        # data in the file.
        work_piece <- list(dataset_type = dataset_type,
                           filename = replaceVariablesInString("$EXP_FULL_PATH$", replace_values),
                           namevar = namevar, is_2d_var = is_2d_var, grid = grid,
                           remap = remap,
                           is_file_per_member = is_file_per_member_exp[jmod],
                           lon_limits = c(lonmin, lonmax),
                           lat_limits = c(latmin, latmax))
        found_dims <- LoadDataFile(work_piece, explore_dims = TRUE, silent = silent)
        if (!is.null(found_dims)) {
          if (is.null(grid) && (output != 'areave')) {
            grid <- replaceVariablesInString(dataset_info[3], replace_values)
          }
          if (is.null(nmember)) {
            nmember <- rep(found_dims[['nmemb']], nmod)
          }
          if (is.null(nleadtime)) {
            nleadtime <- found_dims[['nltime']]
          }
<<<<<<< HEAD
        }
        
        ## We obtain the names of all data files in the directory of the 
        ## repository.
        argfiles <- paste('ls ', 
                    ## First part
                    switch(mod, 
                    'EnsEcmwfDec' = paste(roota, 'ecmwf/2005/s001/m001', 
                                          sep = ""), 
                    'EnsCerfacsDec' = paste(roota, 'cerfacs/2002/s000/m001', 
                                            sep = ""), 
                    'EnsIfmDec' = paste(roota, 'ifm/2002/s001/m010', sep = ""), 
                    'EnsUkmoDec' = paste(roota, 'ukmo/2026/s001/m001', 
                                         sep = ""), 
                    'EnsEcmwfSeas' = paste(rootc, '/seasonal/ecmwf', sep = ""),
                    'EnsCerfacsSeas' = paste(rootc, 'seasonal/cerfacs', 
                                             sep = ""),
                    'EnsIfmSeas' = paste(rootc, 'seasonal/ifm', sep = ""),
                    'EnsUkmoSeas' = paste(rootc, 'seasonal/ukmo', sep = ""),
                    'EnsCmccSeas' = paste(rootc, '/seasonal/cmcc', sep = ""),
                    'EnsMetfrSeas' = paste(rootc, '/seasonal/metfr', sep = ""),
                    'ecmwf' = paste(rootc, 'ecmwf/2005/s001/m001', sep = ""),
                    'cerfacs' = paste(roota, 'cerfacs/2002/s000/m001', 
                                      sep = ""),
                    'ifm' = paste(roota, 'ifm/2002/s001/m010', sep = ""),
                    'ukmo' = paste(roota, 'ukmo/2026/s001/m001', sep = ""),
                    'ECMWF_S3Seas' = rootg,
                    'ECMWF_S4_sea' = paste(rootf, 'm001', sep = ""),
                    'ECMWF_S4_ann' = paste(rootf, 'm003', sep = ""),
                    'bccdec' = paste(rootd, 'bcc/i1p1', sep = ""),
                    'hadcm3dec' = paste(rootd, 'hadcm3/i2p1', sep = ""),
                    'miroc4dec' = paste(rootd, 'miroc4h/i1p1', sep = ""),
                    'miroc5dec' = paste(rootd, 'miroc5/i1p1', sep = ""),
                    'mri-cgcm3dec' = paste(rootd, 'mri-cgcm3/i1p1', sep = ""),
                    'cnrm-cm5dec' = paste(rootd, 'cnrm-cm5/i1p1', sep = ""),
                    'cancm4dec1' = paste(rootd, 'cancm4/i1p1', sep = ""),
                    'cancm4dec2' = paste(rootd, 'cancm4/i2p1', sep = ""),
                    'knmidec' = paste(rootd, 'ecearth/i1p1', sep = ""),
                    'smhidec' = paste(rootd, 'ecearth/i3p1', sep = ""), 
                    'mpimdec' = paste(rootd, 'mpi-esm-lr/i1p1', sep = ""),
                    'mpimdec2' = paste(rooth, 'mpi-esm-lr/i1p1', sep = ""),
                    'gfdldec' = paste(rootd, 'gfdl/i1p1', sep = ""),
                    'cmcc-cmdec' = paste(rootd, 'cmcc-cm/i1p1', sep = ""),
                    'ipsldec' = paste(rootd, 'ipsl/i1p1', sep = ""),
                    'ipsldec2' = paste(rooth, 'ipsl/i1p1', sep = ""),
                    'bcchis' = paste(roote, 'bcc/i0p1', sep = ""),
                    'hadcm3his' = paste(roote, 'hadcm3/i0p1', sep = ""),
                    'miroc4his' = paste(roote, 'miroc4h/i0p1', sep = ""),
                    'miroc5his' = paste(roote, 'miroc5/i0p1', sep = ""),
                    'mri-cgcm3his' = paste(roote, 'mri-cgcm3/i0p1', sep = ""),
                    'cnrm-cm5his' = paste(roote, 'cnrm-cm5/i0p1', sep = ""),
                    'knmihis' = paste(roote, 'ecearth/iXpY', sep = ""),
                    'cancm4his' = paste(roote, 'cancm4/i0p1', sep= ""),
                    'mpimhis' = paste(roote, 'mpi-esm-lr/i0p1', sep = ""),
                    'gfdlhis' = paste(roote, 'gfdl/i0p1', sep = ""),
                    'ipslhis' = paste(roote, 'ipsl/i1p1', sep = ""),
                    'cmcc-cmhis' = paste(roote, 'cmcc-cm/i0p1', sep = ""),
                    'CMC1-CanCM3'= paste(rooti, 'CMC1-CanCM3', sep = ""),
                    'CMC2-CanCM4'= paste(rooti, 'CMC2-CanCM4', sep = ""),
                    'COLA-RSMAS-CCSM3'= paste(rooti, 'COLA-RSMAS-CCSM3', sep = ""),
                    'COLA-RSMAS-CCSM4'= paste(rooti, 'COLA-RSMAS-CCSM4', sep = ""),
                    'GFDL-CM2p1'= paste(rooti, 'GFDL-CM2p1', sep = ""),
                    'GFDL-CM2p1-aer04'= paste(rooti, 'GFDL-CM2p1-aer04', sep = ""),
                    'GFDL-CM2p5-FLOR-B01'=paste(rooti, 'GFDL-CM2p5-FLOR-B01', sep = ""),
                    'GFDL-CM2p5-FLOR-A06'=paste(rooti, 'GFDL-CM2p5-FLOR-A06', sep = ""),
                    'IRI-ECHAM4p5-AnomalyCoupled'=paste(rooti, 'IRI-ECHAM4p5-AnomalyCoupled', sep = ""),
                    'IRI-ECHAM4p5-DirectCoupled'=paste(rooti, 'IRI-ECHAM4p5-DirectCoupled', sep = ""),
                    'NASA-GMAO'=paste(rooti, 'NASA-GMAO', sep = ""),
                    'NASA-GMAO-062012'=paste(rooti, 'NASA-GMAO-062012', sep = ""),
                    paste(rootb, mod, sep = "")), 
                    ## Second part
                    '/', storefreq, '_', dirtype, '/', dirin, 
                    ## Third part
                    switch(mod, 'ecmwf' = '/', 'cerfacs' = '/', 
                    'ifm' = '/', 'ukmo' = '/', 'EnsEcmwfDec' = '/', 
                    'EnsCerfacsDec' = '/', 'EnsUkmoDec' =  '/', 
                    'EnsIfmDec' = '/', 'EnsEcmwfSeas' = '/', 
                    'EnsCerfacsSeas' = '/', 'EnsUkmoSeas' = '/', 
                    'EnsIfmSeas' = '/', 'EnsCmccSeas' = '/', 
                    'EnsMetfrSeas' = '/','CMC1-CanCM3'= '/',
                    'CMC2-CanCM4'= '/', 'COLA-RSMAS-CCSM3'='/',
                    'COLA-RSMAS-CCSM4'='/','GFDL-CM2p1'='/',
                    'GFDL-CM2p1-aer04'='/','GFDL-CM2p5-FLOR-A06'='/',
                    'GFDL-CM2p5-FLOR-B01'='/','IRI-ECHAM4p5-AnomalyCoupled'='/',
                    'IRI-ECHAM4p5-DirectCoupled'='/','NASA-GMAO'='/',
                    'NASA-GMAO-062012'='/', 
                    'ECMWF_S3Seas' = switch(var, 'tos' = '_24hourly/',
                                            'tas' = '_6hourly/', '/'),
                    'ECMWF_S4_sea' = switch(var, 'tos' = '_24hourly/', 
                    '_6hourly/'), 
                    'ECMWF_S4_ann' = switch(var, 'prlr' = '/', 
                                            'tos' = '_24hourly/', '_6hourly/'),
                    'bccdec' = '/', 'hadcm3dec' = '/', 'miroc4dec' = '/', 
                    'miroc5dec' = '/', 'mri-cgcm3dec' = '/', 
                    'cnrm-cm5dec' = '/', 'cancm4dec1' = '/', 'cancm4dec2' = '/',
                    'knmidec' = '/', 'smhidec' = '/', 'bcchis' = '/', 
                    'hadcm3his' = '/', 'miroc4his' = '/', 'miroc5his' = '/', 
                    'mri-cgcm3his' = '/', 'cancm4his' = '/', 
                    'cnrm-cm5his' = '/', 'knmihis' = '/', 'mpimdec' = '/',
                    'mpimdec2' = '/', 'ipsldec2' = '/',
                    'gfdldec' = '/', 'mpimhis' = '/', 'gfdlhis' = '/', 
                    'cmcc-cmdec' = '/', 'cmcc-cmhis' = '/', 'ipsldec' = '/',
                    'ipslhis' = '/', freq),
                    ## Last part
                    file2load, sep = "")
        files <- system(argfiles, intern = T, ignore.stderr = T)
        
        ## If we do not find any file there, we print a warning and set as file
        ## name 'nofile.nc'. If we find any, we take the last as file name.
        ## Then we set the grid. If the grid to apply is the default grid of  
        ## the experiment we do not apply changes to the data file, so the file 
        ## name remains the same.
        ## If the grid has to be changed, we use 'cdo' for this purpose and 
        ## then save the changes into a temporary file named 'tmp.nc'.
        if (length(files) > 0) {
          filename <- files[length(files)]
          print(filename)        
          if (defgrid) {
            grid <- switch(mod, 
                           'ecmwf' = 'r144x73', 
                           'cerfacs' = 'r144x73',
                           'ukmo' = 'r144x73',
                           'ifm' = 'r144x73',
                           'EnsEcmwfDec' = 'r144x73',
                           'EnsCerfacsDec' = 'r144x73',
                           'EnsUkmoDec' = 'r144x73',
                           'EnsIfmDec' = 'r144x73',
                           'EnsEcmwfSeas' = 'r144x73',
                           'EnsCerfacsSeas' = 'r144x73',
                           'EnsUkmoSeas' = 'r144x73',
                           'EnsIfmSeas' = 'r144x73',
                           'EnsCmccSeas' = 'r144x73',
                           'EnsMetfrSeas' = 'r144x73',
                           'ECMWF_S4_sea' = 't170grid',
                           'ECMWF_S4_ann' = 't170grid',
                           'bccdec' = 't42grid',
                           'hadcm3dec' = switch(var, 
                                                'tos' = 'r288x144',
                                                'tas' = 'r96x73',
                                                'prlr' = 'r96x73',
                                                'psl' = 'r96x73'),
                           'miroc4dec' = 'r360x180',
                           'miroc5dec' = 't85grid',
                           'mri-cgcm3dec' = switch(var,
                                                   'tos' = 'r240x180',
                                                   'tas' = 't106grid',
                                                   'prlr' = 't106grid',
                                                   'psl' = 't106grid'),
                           'cnrm-cm5dec' = 't85grid',
                           'cancm4dec1' = switch(var,
                                                 'tas' = 't42grid',
                                                 'prlr' = 't42grid',
                                                 'psl' = 't42grid'),
                           'cancm4dec2' = switch(var,
                                                 'tas' = 't42grid',
                                                 'prlr' = 't42grid'),
                           'knmidec' = 'r320x161',
                           'mpimdec' = 't63grid',
                           'mpimdec2' = 't63grid',
                           'gfdldec' = switch(var,
                                              'tas' = 'r144x90',
                                              'prlr' = 'r144x90',
                                              'tos' = 'r144x90'),
                           'cmcc-cmdec' = switch(var,
                                                 'tas' = 't159grid',
                                                 'prlr' = 't159grid'),
                           'ipsldec' = 'r96x96',
                           'ipsldec2' = 'r96x96',
                           'bcchis' = 't42grid',
                           'hadcm3his' = switch(var,
                                                'tos' = 'r288x144',
                                                'tas' = 'r96x73',
                                                'prlr' = 'r96x73',
                                                'psl' = 'r96x73'),
                           'miroc4his' = 'r360x180',
                           'miroc5his' = 't85grid',
                           'mri-cgcm3his' = switch(var,
                                                   'tos' = 'r240x180',
                                                   'tas' = 't106grid',
                                                   'prlr' = 't106grid',
                                                   'psl' = 't106grid'),
                           'cnrm-cm5his' = 't85grid',
                           'cancm4his' = switch(var,
                                                'tas' = 't42grid',
                                                'prlr' = 't42grid',
                                                'psl' = 't42grid'),
                           'mpimhis' = 't63grid',
                           'gfdlhis' = switch(var,
                                              'tas' = 'r144x90',
                                              'prlr' = 'r144x90',
                                              'tos' = 'r144x90'),
                           'ipslhis' = 'r96x96',
                           'cmcc-cmhis' = 't159grid',
                           switch(var,
                                  'ta50' = 't159grid',
                                  't106grid'))
            filein <- filename
          } else {
            system(paste("cdo ", remap, ",", grid, " -selname,", namevar, " ", filename, 
                         " tmp.nc 2>/dev/null", sep = ""))
            system("ncrename -v lat,latitude -d lat,latitude -v lon,longitude -d lon,longitude tmp.nc")
            filein <- "tmp.nc"
=======
          if (is.null(leadtimemax)) {
            leadtimemax <- nleadtime
          } else if (leadtimemax > nleadtime) {
            stop("Error: 'leadtimemax' argument is greater than the number of loadedleadtimes. 
                  Put first the experiment with the greatest number of leadtimes or adjust the parameters 'nleadtime' and 'leadtimemax' properly.")
>>>>>>> 00243c3acb14d45481699c234ef88919310f51ed
          }
          leadtimes <- seq(leadtimemin, leadtimemax, sampleperiod)
          latitudes <- found_dims[['lat']]
          longitudes <- found_dims[['lon']]
          
          if (output == 'lon' || output == 'lonlat') {
            dim_exp[['lon']] <- length(longitudes)
          }
          if (output == 'lat' || output == 'lonlat') {
            dim_exp[['lat']] <- length(latitudes)
          }
          dim_exp[['nltime']] <- length(leadtimes)
          dim_exp[['nmember']] <- max(nmember)
          dim_exp[['nsdates']] <- nsdates
          dim_exp[['ndat']] <- nmod
          dims2define <- FALSE
        }
      }
      # We keep on iterating through members to build all the work pieces.
      if (is_file_per_member_exp[jmod]) {
        jmember <- 1
        while (jmember <= nmember[jmod]) {
          replace_values <- putInDictionary(replace_values, "MEMBER_NUMBER", 
                            sprintf(paste("%.", (nmember[jmod] %/% 10) + 1, "i", sep = ''), jmember - 1))
          work_piece <- list(filename = replaceVariablesInString("$EXP_FULL_PATH$", replace_values),
                             namevar = namevar, indices = c(1, jmember, jsdate, jmod), 
                             nmember = nmember[jmod], leadtimes = leadtimes, mask = maskmod[[jmod]],
                             is_file_per_dataset = is_file_per_dataset_exp[jmod],
                             var_limits = c(mod_var_min, mod_var_max))
          exp_work_pieces <- c(exp_work_pieces, list(work_piece))
          jmember <- jmember + 1
        }
      } else {
        work_piece <- list(filename = replaceVariablesInString("$EXP_FULL_PATH$", replace_values),
                           namevar = namevar, indices = c(1, 1, jsdate, jmod), 
                           nmember = nmember[jmod], leadtimes = leadtimes, mask = maskmod[[jmod]],
                           is_file_per_dataset = is_file_per_dataset_exp[jmod],
                           var_limits = c(mod_var_min, mod_var_max))
        exp_work_pieces <- c(exp_work_pieces, list(work_piece))
      }
      jsdate <- jsdate + 1
    }
    jmod <- jmod + 1
  }
  if (dims2define && length(exp) > 0) {
    print("Warning: no data found in file system for any experimental dataset. Forcing 'exp' to NULL.")
    exp <- NULL
    dims2define <- FALSE
  }
  # If there are no experiments to load we need to choose a number of time steps
  # to load from observational datasets. We load from the first start date to 
  # the current date.
  if (is.null(exp)) {
    diff <- Sys.time() - as.POSIXct(paste(substr(sdates[1], 1, 4), '-',
            substr(sdates[1], 5, 6), '-', substr(sdates[1], 7, 8), sep=''))
    if (storefreq == 'monthly') { 
      leadtimemax <- as.integer(diff/30) 
    } else {
      leadtimemax <- as.integer(diff)
    }
    if (is.null(nleadtime)) {
      nleadtime <- leadtimemax
    }
    leadtimes <- seq(leadtimemin, leadtimemax, sampleperiod)
  }

  if (!silent) {
    message <- paste("* Success: ", nmod, ", ", 
                     ifelse(is.null(nmember), 0, max(nmember)), ", ", 
                     nsdates, ", ", length(leadtimes), sep = '')
    if (output == 'lon') {
      message <- paste(message, ", ", length(longitudes), sep = '')
    } else if (output == 'lat') {
      message <- paste(message, ", ", length(longitudes), sep = '')
    } else if (output == 'lonlat') {
      message <- paste(message, ", ", length(latitudes), ", ", length(longitudes), sep = '')
    }
    print(message)
    print("* Fetching first observational files to work out 'var_obs' size...")
  }
  
  # Now we start iterating over observations. We try to find the output matrix
  # dimensions and we build anyway the work pieces corresponding to the observational
  # data that time-corresponds the experimental data or the time-steps until the
  # current date if no experimental datasets were specified.
  dataset_type <- 'obs'
  dim_obs <- NULL
  dims2define <- TRUE
  lat_indices <- lon_indices <- NULL
  obs_work_pieces <- list()
  jobs <- 1
  while (jobs <= nobs) {
    replace_values <- putInDictionary(replace_values, "OBS_NAME", obs[jobs])
    dataset_info <- obs_info[[jobs]]
    replace_values <- putInDictionary(replace_values, "OBS_MAIN_PATH", dataset_info[1])
    replace_values <- putInDictionary(replace_values, "OBS_FILE_PATH", dataset_info[2])
    replace_values <- putInDictionary(replace_values, "NC_VAR_NAME", dataset_info[3])
    namevar <- replaceVariablesInString(dataset_info[3], replace_values)
    if (is.null(suffixobs) || is.na(suffixobs[jobs])) {
      replace_values <- putInDictionary(replace_values, "SUFFIX", dataset_info[4])
    } else {
      replace_values <- putInDictionary(replace_values, "SUFFIX", suffixobs[jobs])
    }
    if (is.null(varmin)) {
      obs_var_min <- as.numeric(replaceVariablesInString(dataset_info[5], replace_values))
    } else {
      obs_var_min <- varmin
    }
    if (is.null(varmax)) {
      obs_var_max <- as.numeric(replaceVariablesInString(dataset_info[6], replace_values))
    } else {
      obs_var_max <- varmax
    }
    # This file format (file per whole dataset) is only supported in observations.
    # However a file per whole dataset experiment could be seen as a file per
    # member/ensemble experiment with a single start date, so still loadable.
    # Nonetheless file per whole dataset observational files do not need to contain
    # a year and month in the filename, the time correspondance relies on the 
    # month and years associated to each timestep inside the NetCDF file.
    # So file per whole dataset experiments need to have a start date in the filename.
    if (is_file_per_dataset_obs[jobs]) {
      ## TODO: Open file-per-dataset-files only once.
      if (dims2define) {
        work_piece <- list(dataset_type = dataset_type,
                           filename = replaceVariablesInString("$OBS_FULL_PATH$", replace_values),
                           namevar = namevar, is_2d_var = is_2d_var, grid = grid,
                           remap = remap, is_file_per_member = is_file_per_member_obs[jobs],
                           lon_limits = c(lonmin, lonmax),
                           lat_limits = c(latmin, latmax))
        found_dims <- LoadDataFile(work_piece, explore_dims = TRUE, silent = silent)
        if (!is.null(found_dims)) {
          if (is.null(nmemberobs)) {
            nmemberobs <- rep(found_dims[['nmemb']], nobs)
          }
          if (is.null(exp)) {
            longitudes <- found_dims[['lon']]
            latitudes <- found_dims[['lat']]
          }
          
          if (output == 'lon' || output == 'lonlat') {
            dim_obs[['lon']] <- length(longitudes)
          }
          if (output == 'lat' || output == 'lonlat') {
            dim_obs[['lat']] <- length(latitudes)
          }
          dim_obs[['nltime']] <- length(leadtimes)
          dim_obs[['nmember']] <- max(nmemberobs)
          dim_obs[['nsdates']] <- nsdates
          dim_obs[['ndat']] <- nobs
          dims2define <- FALSE
        }
      }
    
      work_piece <- list(filename = replaceVariablesInString("$OBS_FULL_PATH$", replace_values),
                         namevar = namevar, indices = c(1, 1, 1, jobs), 
                         nmember = nmemberobs[jobs], 
                         mask = maskobs[[jobs]], leadtimes = leadtimes, 
                         is_file_per_dataset = is_file_per_dataset_obs[jobs], 
                         startdates = sdates, var_limits = c(obs_var_min, obs_var_max))
      obs_work_pieces <- c(obs_work_pieces, list(work_piece))
    } else {
      jsdate <- 1
      while (jsdate <= nsdates) {
        replace_values <- putInDictionary(replace_values, "START_DATE", sdates[jsdate])         
        sdate <- sdates[jsdate]

        if (storefreq == 'daily') {
          day <- substr(sdate, 7, 8)
          if (day == '') {
            day <- '01'
          }
          day <- as.integer(day)
          startdate <- as.POSIXct(paste(substr(sdate, 1, 4), '-',
                       substr(sdate, 5, 6), '-', day, ' 12:00:00', sep = '')) + 
                       (leadtimemin - 1) * 86400
          year <- as.integer(substr(startdate, 1, 4))
          month <- as.integer(substr(startdate, 6, 7))
        } else {
          month <- (as.integer(substr(sdate, 5, 6)) + leadtimemin - 2) %% 12 + 1
          year <- as.integer(substr(sdate, 1, 4)) + (as.integer(substr(sdate, 
                  5, 6)) + leadtimemin - 2) %/% 12
        }
        jleadtime <- 1
        while (jleadtime <= length(leadtimes)) {
          replace_values <- putInDictionary(replace_values, "YEAR", paste(year, '', sep = ''))
          replace_values <- putInDictionary(replace_values, "MONTH", sprintf("%2.2i", month))
        
          if (storefreq == 'daily') {
            days_in_month <- ifelse(LeapYear(year), 29, 28)
            days_in_month <- switch(paste(month, '', sep = ''), '1' = 31, 
                                    '3' = 31, '4' = 30, '5' = 31, '6' = 30, 
                                    '7' = 31, '8' = 31, '9' = 30, '10' = 31, 
                                    '11' = 30, '12' = 31, days_in_month)
            ## This condition must be fulfilled to put all the month time steps
            ## in the dimension of length nleadtimes. Otherwise it must be cut:
            #(length(leadtimes) - 1)*sampleperiod + 1 - (jleadtime - 1)*sampleperiod >= days_in_month - day + 1
            obs_file_indices <- seq(day, min(days_in_month, (length(leadtimes) - jleadtime) * sampleperiod + day), sampleperiod)
          } else {
            obs_file_indices <- 1
          }
          if (dims2define) {
            if (is_file_per_member_obs[jobs]) {
              replace_values <- putInDictionary(replace_values, "MEMBER_NUMBER", '*')
            }
            work_piece <- list(dataset_type = dataset_type,
                               filename = replaceVariablesInString("$OBS_FULL_PATH$", replace_values),
                               namevar = namevar, is_2d_var = is_2d_var, grid = grid,
                               remap = remap,
                               is_file_per_member = is_file_per_member_obs[jobs],
                               lon_limits = c(lonmin, lonmax),
                               lat_limits = c(latmin, latmax))
            found_dims <- LoadDataFile(work_piece, explore_dims = TRUE, silent = silent)
            if (!is.null(found_dims)) {
              if (is.null(nmemberobs)) {
               nmemberobs <- rep(found_dims[['nmemb']], nobs)
              }
              if (is.null(exp)) {
                longitudes <- found_dims[['lon']]
                latitudes <- found_dims[['lat']]
              }
              
              if (output == 'lon' || output == 'lonlat') {
                dim_obs[['lon']] <- length(longitudes)
              }
              if (output == 'lat' || output == 'lonlat') {
                dim_obs[['lat']] <- length(latitudes)
              }
              dim_obs[['nltime']] <- length(leadtimes)
              dim_obs[['nmember']] <- max(nmemberobs)
              dim_obs[['nsdates']] <- nsdates
              dim_obs[['ndat']] <- nobs
              dims2define <- FALSE
            }
          }
          if (is_file_per_member_obs[jobs]) {
            jmember <- 1
            while (jmember <= nmemberobs[jobs]) {
              replace_values <- putInDictionary(replace_values, "MEMBER_NUMBER", 
                                sprintf(paste("%.", (nmemberobs[jobs] %/% 10) + 1, "i", sep = ''), jmember - 1))
              work_piece <- list(filename = replaceVariablesInString("$OBS_FULL_PATH$", replace_values),
                                 namevar = namevar, indices = c(jleadtime, jmember, jsdate, jobs), 
                                 nmember = nmemberobs[jobs], leadtimes = obs_file_indices, 
                                 mask = maskobs[[jobs]],
                                 is_file_per_dataset = is_file_per_dataset_obs[jobs], 
                                 var_limits = c(obs_var_min, obs_var_max))
              obs_work_pieces <- c(obs_work_pieces, list(work_piece))
              jmember <- jmember + 1
            }
          } else {
            work_piece <- list(filename = replaceVariablesInString("$OBS_FULL_PATH$", replace_values),
                               namevar = namevar, indices = c(jleadtime, 1, jsdate, jobs), 
                               nmember = nmemberobs[jobs], leadtimes = obs_file_indices, 
                               mask = maskobs[[jobs]],
                               is_file_per_dataset = is_file_per_dataset_obs[jobs], 
                               var_limits = c(obs_var_min, obs_var_max))
            obs_work_pieces <- c(obs_work_pieces, list(work_piece))
          }
          
          if (storefreq == 'daily') {
            startdate <- startdate + 86400 * sampleperiod * length(obs_file_indices)
            year <- as.integer(substr(startdate, 1, 4))
            month <- as.integer(substr(startdate, 6, 7))
            day <- as.integer(substr(startdate, 9, 10))
          } else {
            month <- month + sampleperiod
            year <- year + (month - 1) %/% 12
            month <- (month - 1) %% 12 + 1
          }
          jleadtime <- jleadtime + length(obs_file_indices)
        }
        
        jsdate <- jsdate + 1
      }
    }
    jobs <- jobs + 1
  }
  if (dims2define && length(obs) > 0) {
    print("Warning: no data found in file system for any observational dataset. Forcing 'obs' to NULL.")
    obs <- NULL
    dims2define <- FALSE
  }

  if (!silent) {
    message <- paste("* Success: ", nobs, ", ", 
                     ifelse(is.null(nmemberobs), 0, max(nmemberobs)), ", ", 
                     nsdates, ", ", length(leadtimes), sep = '')
    if (output == 'lon') {
      message <- paste(message, ", ", length(longitudes), sep = '')
    } else if (output == 'lat') {
      message <- paste(message, ", ", length(latitudes), sep = '')
    } else if (output == 'lonlat') {
      message <- paste(message, ", ", length(latitudes), ", ", 
                                      length(longitudes), sep = '')
    }
    print(message)
  }

  # We buil two matrices in shared memory for the parallel processes to
  # store their results
  # These matrices will contain data arranged with the following
  # dimension order, to maintain data spacial locality during the 
  # parallel fetch:
  #   longitudes, latitudes, leadtimes, members, startdates, nmod/nobs
  # So [1, 1, 1, 1, 1, 1] will be next to [2, 1, 1, 1, 1, 1] in memory
  pointer_var_exp <- NULL
  if (!is.null(exp)) {
    var_exp <- big.matrix(nrow = prod(dim_exp), ncol = 1)
    pointer_var_exp <- describe(var_exp)
  }
  pointer_var_obs <- NULL
  if (!is.null(obs)) {
    var_obs <- big.matrix(nrow = prod(dim_obs), ncol = 1)
    pointer_var_obs <- describe(var_obs)
  }

  if (is.null(nprocs)) {
    nprocs <- detectCores()
  }

  ## TODO: Try 2 cores only
  # Add some important extra fields in the work pieces before sending
  exp_work_pieces <- lapply(exp_work_pieces, function (x) c(x, list(dataset_type = 'exp', dims = dim_exp, out_pointer = pointer_var_exp)))
  obs_work_pieces <- lapply(obs_work_pieces, function (x) c(x, list(dataset_type = 'obs', dims = dim_obs, out_pointer = pointer_var_obs)))
  work_pieces <- c(exp_work_pieces, obs_work_pieces)
  if (!silent) {
    print(paste("* Reading and processing data files (on ", nprocs, " parallel processes):", sep = ''))
  }
  work_pieces <- lapply(work_pieces, 
                 function (x) {
                   if (!silent) {
                     print(paste("*   ", x[['filename']], sep = ''))
                   }
                   c(x, list(is_2d_var = is_2d_var, grid = grid, remap = remap,  
                             lon_limits = c(lonmin, lonmax), 
                             lat_limits = c(latmin, latmax), output = output))
                 })
  # Build the cluster of processes that will do the work and dispatch work pieces.
  # The function LoadDataFile is applied to each work package. This function will
  # open the data file, regrid if needed, trim (select time steps, longitudes, 
  # latitudes, members), apply the mask, compute and apply the weights if needed,
  # disable extreme values and store in the shared memory matrices. 
  if (nprocs == 1) {
    found_files <- lapply(work_pieces, LoadDataFile, silent = silent)
  } else {
    cluster <- makeCluster(nprocs)
    found_files <- clusterApply(cluster, work_pieces, LoadDataFile, silent = silent)
    stopCluster(cluster)
  }
  if (!silent && any(!unlist(found_files))) {
    print("* Warning: The following files were not found in the file system. Filling with NA values instead:")
    lapply(work_pieces[!unlist(found_files)], function (x) print(paste("*   ", x[['filename']], sep = '')))
  }

  # Before ending, the data is arranged in the common format, with the following
  # dimension order:
  #  nmod/nobs, members, startdates, leadtimes, latitudes, longitudes
  invisible(list(mod = if (is.null(exp)) {
                         NULL
                       } else { 
                         dim_reorder <- length(dim_exp):1
                         dim_reorder[2:3] <- dim_reorder[3:2]
                         aperm(array(bigmemory::as.matrix(var_exp), dim = dim_exp), dim_reorder)
                       }, 
                 obs = if (is.null(obs)) {
                         NULL
                       } else {
                         dim_reorder <- length(dim_obs):1
                         dim_reorder[2:3] <- dim_reorder[3:2]
                         aperm(array(bigmemory::as.matrix(var_obs), dim = dim_obs), dim_reorder)
                       }, 
                 lat = latitudes, lon = longitudes))
}
