# Author: An-Chi Ho
# Date: 7th Oct. 2020
#---------------------------------------------------------------------
# The script shows three ways to load the data with different expid - member - version
# combination. It is useful for climate prediction of multiple experiments.
# In this case, the two datasets have the following combination:
# | expid | member   | version |
# |-------|----------|---------|
# | a1st  | r7i1p1f1 |v20190302|
# | a1sx  |r10i1p1f1 |v20190308|
#
# The three methods to load the data are:
# (1) dependencies
# (2) glob expression
# (3) dataset
#---------------------------------------------------------------------

library(startR)

# (1) dependencies
# Because the three file dimensions 'expid', 'member', and 'version' have dependency  
# on each other, so we can use the parameter 'xxx_depends' to specify the relationship.

repos <- paste0('/esarchive/exp/ecearth/$expid$/diags/CMIP/EC-Earth-Consortium/',
                'EC-Earth3/historical/$member$/Omon/$var$/gn/$version$/',
                '$var$_Omon_EC-Earth3_historical_$member$_gn_$year$.nc')
yrh1 <- 1960
yrh2 <- 1961
years  <- paste0(c(yrh1 : yrh2), '01-', c(yrh1 : yrh2), '12')

data <- Start(dat = repos,
              var = 'tosmean',
              expid =  c('a1st', 'a1sx'),
              member = 'all',
              version =  'all',
              member_depends = 'expid',
              member_depends = 'version',
              version_depends = 'expid',
              version_depends = 'member',  
              year = years,
              time = 'all',
              region = 'all',
              return_vars = list(time = NULL, region = NULL),
              retrieve = T)

dim(data)
    dat     var   expid  member version    year    time  region 
      1       1       2       1       2       2      12      14 


# (2) glob expression
# The parameter 'path_glob_permissive' allows to use '*' to define the path.
# Note that '*' can only represent one possibility. Because each expid only has
# one member and one version, so we can use '*' to define the path.
# See Start() documentation for more details of 'path_glob_permissive'.

repos <- paste0('/esarchive/exp/ecearth/$expid$/diags/CMIP/EC-Earth-Consortium/',
                'EC-Earth3/historical/*/Omon/$var$/gn/v*/',
                '$var$_Omon_EC-Earth3_historical_*_gn_$year$.nc')

yrh1 <- 1960
yrh2 <- 1961
years  <- paste0(c(yrh1 : yrh2), '01-', c(yrh1 : yrh2), '12')

data <- Start(dat = repos,
              var = 'tosmean',
              expid =  c('a1st', 'a1sx'),
              year = years,
              time = 'all',
              region = 'all',
              path_glob_permissive = 6, # to preserve * for the last 6 folder layers (6th is $member$ originally)
              return_vars = list(time = NULL, region = NULL),
              retrieve = T)

dim(data)
   dat    var  expid   year   time region 
     1      1      2     55     12     14 


# (3) dataset
# We can simply define two expID as two datasets. Therefore, the member and version 
# can be specified in each path directly. 
# The following script is a bit different from the above two. It read two versions
# for the first dataset. Therefore, the result dimension 'version = 2', and the second 
# dataset has NAs along [version = 2] since it only has one version.

repos <- list(list(name = 'a1st',
                   path = paste0('/esarchive/exp/ecearth/a1st/diags/CMIP/EC-Earth-Consortium/',
                                  'EC-Earth3/historical/r7i1p1f1/Omon/$var$/gn/$version$/',
                                 '$var$_Omon_EC-Earth3_historical_r7i1p1f1_gn_$year$.nc')),
              list(name = 'a1sx',
                    path = paste0('/esarchive/exp/ecearth/a1sx/diags/CMIP/EC-Earth-Consortium/',
                                 'EC-Earth3/historical/r10i1p1f1/Omon/$var$/gn/$version$/',
                                 '$var$_Omon_EC-Earth3_historical_r10i1p1f1_gn_$year$.nc'))
             )
yrh1 <- 1960
yrh2 <- 1961 #2014
years  <- paste0(c(yrh1 : yrh2), '01-', c(yrh1 : yrh2), '12')

data <- Start(dat = repos,
              var = 'tosmean',
              year = years,
              version = indices(1:2), #'all',
              time = 'all',
              region = 'all',
              retrieve = T)

 dim(data)
    dat     var    year version    time  region 
      2       1       2       2      12      14  
