# Author: An-Chi Ho
# Date: 17th Aug. 2020
#---------------------------------------------------------------------
# This script shows you how to use glob expression '*' and the parameter 'path_glob_permissive' of Start().
# The regular way to define the path for Start() is using the tags, i.e., '$<dim_name>$'. 
# However, we may want to use '*' to define the path under some conditions.
# Start() allows this usage but with some limitation. You can find more details in Start() documentation, the parameter 'path_glob_permissive'.
#---------------------------------------------------------------------
library(startR)

#-----------------------------------------------------------------
# Case 1: Use * to substitute a string which is common among files
#-----------------------------------------------------------------
## This is a lazy way to define the path. When there is a common string among all
## the files, you can use * instead of the string. 
## However, it is not recommended to use because you may not notice when the file 
## path is changed and Start() reads the unwanted files. The more clear-defined the
## path, the safer it is.

## We want to read the two files, 1960 and 1961, from the repository. The full path
## definition is below.
repo_original <- paste0('/esarchive/exp/ecearth/a1st/diags/CMIP/EC-Earth-Consortium/',
                        'EC-Earth3/historical/r7i1p1f1/Omon/$var$/gn/v20190302/',
                        '$var$_Omon_EC-Earth3_historical_r7i1p1f1_gn_$year$.nc')
years  <- paste0(c(1960:1961), '01-', c(1960:1961), '12')

## We know that there are some layers of the path are unique and identical between
## the two files. We can change those layers to *.
## Please note that if the string is different between files, * must be used with the 
## parameter 'path_glob_permissive' (See Case 2 below).
repos <- paste0('/esarchive/exp/ecearth/a1st/diags/CMIP/*/',
                '*/*/r7i1p1f1/Omon/$var$/*/v20190302/',
                '$var$_Omon_*_*_r7i1p1f1_*_$year$.nc')

data <- Start(dat = repos,
              var = 'tosmean',
              year = years,
              time = indices(1),
              region = indices(1),
              return_vars = list(time = NULL, region = NULL),
              retrieve = T)


#-------------------------------------------------------------
# Case 2: Use * to substitute a name which differs from files
#------------------------------------------------------------- 
## We want to read two experiments together, while their member and version name are 
## different. The data structure is like this:
## -- expid --   -- member --   -- version --
##    a1st         r7i1p1f1        v20190302
##    a1sx         r10i1p1f1       v20190308
##
## We cannot simply substitue 'member' and 'version' with *, because Start() will only detect
## the first file path and use the string in the first file to represent the rest files.
## Therefore, the rest files cannot be found. That is, Start() will regard a1st and a1sx
## both have member 'r7i1p1f1' and version 'v20190302'.
## Fortunately, the expid only has one member and one version each. We can use * combined 
## with the parameter 'path_glob_permissive' to make Start() work.

## The * from the end in order represents: 1. member 2. version 3. member
repos <- paste0('/esarchive/exp/ecearth/$expid$/diags/CMIP/EC-Earth-Consortium/',
                'EC-Earth3/historical/*/Omon/$var$/gn/v*/',
                '$var$_Omon_EC-Earth3_historical_*_gn_$year$.nc')
years  <- paste0(c(1960:1961), '01-', c(1960:1961), '12')

## The parameter 'path_glob_permissive' can be assigned with TRUE/FALSE or an integer.
## TRUE equals to 1, and FALSE equals to 0. The number means that how many layers from 
## the end to preserve * for all files. That is, Start() won't replace * with the string
## found in the first file. Start() will use * to look for the pattern for each file.
## So, since the three * in above path are all different between a1st and a1sx, we need 
## to preserve all of them. The furthest layer from the end is 6, so 'path_glob_permissive'
## should be 6.
data <- Start(dat = repos,
              var = 'tosmean',
              expid =  c('a1st', 'a1sx'),
              year = years,
              time = indices(1),
              region = indices(1),
              path_glob_permissive = 6,
              return_vars = list(time = NULL, region = NULL),
              retrieve = T)

## What if certain expid has more than one 'member' or 'version'? For example,
## -- expid --   -- member --   -- version --
##    a1st         r7i1p1f1        v20190302
##                                 v20200302
##    a1sx         r10i1p1f1       v20190308

## In this case, Start() will only catch 'v20190302' for a1st. This is the main difference
## between * and tag. Tag is the dimension of the output array, so it can save as many 
## value as you want. On the other hand, * is not a dimension, so it can only represent one
## possibility for one file path. 


## You may think of the usage of '_depends', which also tells Start() that one tag (i.e.,
## dimension) is dependent on another one. However, '_depends' can only be used when the
## dependency is between two. In this case, we can define the dependency between 'expid' 
## and 'member', and use * for 'version'.
repos <- paste0('/esarchive/exp/ecearth/$expid$/diags/CMIP/EC-Earth-Consortium/',
                'EC-Earth3/historical/$member$/Omon/$var$/gn/v*/',
                '$var$_Omon_EC-Earth3_historical_*_gn_$year$.nc')

## You can see that the layers contain * are the last two. Therefore, 'path_glob_permissive'
## is 2 here.
data <- Start(dat = repos,
              var = 'tosmean',
              expid =  c('a1st', 'a1sx'),
              year = years,
              member = indices(1),
              member_depends = 'expid',
              time = indices(1),
              region = indices(1),
              path_glob_permissive = 2, 
              return_vars = list(time = NULL, region = NULL),
              retrieve = T)

