#---------------------------------------------------------------------
# This usecase shows the things to be noticed when the parameters 
# 'split_multiselected_dims' and 'merge_across_dims' are both used.
# The problem may occur when the dimension number of the splitted selector is 
# more than two. If you're not familiar with the usage of these parameters, 
# please see usecases ex1_2 and ex1_3 first, which are less complicated.
# See FAQ How-to-#17 for more explanation.

# In this example, hindcast data corresponding to 2 start dates ('20160704', '20161222') 
# is retrieved. The hindcasts consist of daily values for the 47 following days 
# after each start date (in this case only the first 12 days are retrieved) and 
# for the 20 years of hindacst (in this case only the first 3 years are retrieved). 
# Once the hindacst data is retrieved/read, the corresponding ERA5 values for each 
# hindcast time and year are retrieved using the 'dates' metadata from the hindcast object.
#---------------------------------------------------------------------

library(startR)

var_name <- 'sfcWind'

# experimental data
path.exp <- '/esarchive/exp/ecmwf/s2s-monthly_ensforhc/daily/$var$_f24h/$sdate$/$var$_$syear$.nc'

hcst <- Start(dat = path.exp,
              var =  var_name,
              sdate = c('20160704', '20161222'),
              syear = indices(1:3),  #1996:1998
              syear_depends = 'sdate',
              time = indices(1:12),  #4th-15th Jul; 22nd Dec-2nd Jan
              latitude = indices(1:10),
              longitude = indices(1:10),
              ensemble = 'all',
              return_vars = list(latitude = 'dat',
                                 longitude = 'dat',
                                 time = c('sdate', 'syear') #time depends on both sdate and syear
                                 ),
              retrieve = F)

#-----------------------------------------------------------------------
# The time attribute is going to be used in retrieving observational data.
# Because 'time' depends on both 'sdate' and 'syear', the dimension number 
# is 3. 

dates <- attr(hcst, 'Variables')$common$time
dim(dates)
#sdate syear  time
#    2     3    12
#-----------------------------------------------------------------------

#-----------------------------------------------------------------------
# If you need to reorder the dimensions of the 'time' selector, you can use
# s2dv::Reorder function. This line is not used in the following example.
#
# dates <- s2dv::Reorder(dates, c('syear', 'sdate', 'time'))
#-----------------------------------------------------------------------

#-----------------------------------------------------------------------
# Use 'dates' generated above to get the required files. 
# The order of file_date is not important. sort() is not necessary to use.
file_date <- sort(unique(gsub('-', '', 
                              sapply(as.character(dates), substr, 1, 7))))
print(file_date)
#[1] "199607" "199612" "199701" "199707" "199712" "199801" "199807" "199812"
#[9] "199901"
#-----------------------------------------------------------------------

# observational data
path.obs <- '/esarchive/recon/ecmwf/era5/1hourly/$var$/$var$_$file_date$.nc'

obs <- Start(dat = path.obs,
             var = var_name,
             file_date = file_date,   # a vector with the information of sdate and syear
             latitude = indices(1:10),
             longitude = indices(1:10),
             time = values(dates),    # a 3-dim array (sdate, syear, time)
             time_across = 'file_date',
             merge_across_dims = TRUE,
             merge_across_dims_narm = TRUE,
             split_multiselected_dims = TRUE,
             synonims = list(latitude = c('lat','latitude'),
                             longitude = c('lon','longitude')),
             return_vars = list(latitude = 'dat',
                                longitude = 'dat',
                                time = 'file_date'),
             retrieve = T)

#----------- Check data ----------------
attr(hcst, 'Dimensions')
#      dat       var     sdate     syear      time  latitude longitude  ensemble 
#        1         1         2         3        12        10        10        11 
dim(obs)
#      dat       var  latitude longitude     sdate     syear      time 
#        1         1        10        10         2         3        12 

# The dimension structure is the same as experimental one.

#------------ Check if the data are placed correctly-----------------
# For example, [sdate = 2, syear = 1, time = 1:12]
dates[2, 1, ]
# [1] "1996-12-22 UTC" "1996-12-23 UTC" "1996-12-24 UTC" "1996-12-25 UTC"
# [5] "1996-12-26 UTC" "1996-12-27 UTC" "1996-12-28 UTC" "1996-12-29 UTC"
# [9] "1996-12-30 UTC" "1996-12-31 UTC" "1997-01-01 UTC" "1997-01-02 UTC"

# obs at [sdate = 2, syear = 1, time = 1:12] should have data for the corresponding times.
# Use ncdf4 package to read the netCDF files and compare.

obs[1, 1, 2, 3, 2, 1, ]
# [1]  4.565837  3.254213  5.710205 10.255745  5.809094  5.477635  3.184075
# [8]  7.230077 14.662762  2.256792  6.470966  5.574388


# NOTE: The following lines require much memory scpace. Run them on interactive session 
#       rather than workstation.
library(ncdf4)
file199612 <- nc_open('/esarchive/recon/ecmwf/era5/1hourly/sfcWind/sfcWind_199612.nc')
wind199612 <- ncvar_get(file199612, 'sfcWind')
file199701 <- nc_open('/esarchive/recon/ecmwf/era5/1hourly/sfcWind/sfcWind_199701.nc')
wind199701 <- ncvar_get(file199701, 'sfcWind')

# The file has 1hr frequency and the dimensions are [lon, lat, time]
# 505 is 1996-12-22 0; 721 is 1996-12-31 0; etc.
wind199612[3, 2, seq(505, 721, by = 24)]
# [1]  4.565837  3.254213  5.710205 10.255745  5.809094  5.477635  3.184075
# [8]  7.230077 14.662762  2.256792

wind199701[3, 2, c(1, 25)]
#[1] 6.470966 5.574388

# The data of retrieved obs and netCDF are identical.
