#---------------------------------------------------------------------
# This usecase shows you how to load experimental and observational data in a 
# consistent way.

# First, load the experimental data. Then, use the time attributes of experimental data to define the selectors for observational data.

# You can see how to use parameter '*_across', 'merge_across_dims', 'merge_across_dims_narm',
# and 'split_multiselected_dims' to create the same dimension structure.
#---------------------------------------------------------------------

# experimental data
# May to December 1994, monthly file with 6-hourly frequency
  repos <- paste0('/esarchive/exp/ecmwf/system4_m1/6hourly/',
                  '$var$/$var$_$sdate$.nc')
  sdate <- sapply(1994, function(x) paste0(x, sprintf('%02d', 5:12), '01'))

  system4 <- Start(dat = repos,
                   var = 'tas',
                   sdate = sdate,
                   time = indices(seq(1, 124, 4)),  #first time step per day
                   ensemble = 'all',
                   latitude = indices(1:10),
                   longitude = indices(1:10),
                   return_vars = list(latitude = NULL,
                                      longitude = NULL,
                                      time = c('sdate')))

#-------- Check exp data -----------
 attr(system4, 'Dimensions')
#      dat       var     sdate      time  ensemble  latitude longitude 
#        1         1         8        31        51        10        10 
#-----------------------------------

# ------- retrieve the attributes for obs load-in ----------
  dates <- attr(system4, 'Variables')$common$time
# dim(dates)
#sdate  time 
#    8    31 

# NOTE: Even though June, September, and November only have 30 days, it reads
#       31 days for each month. Therefore, the last day of these months is the
#       first day of the following month. No NA in this dates array.

# Manually retrieve the observation dates in the required format 
  dates_file <- sort(unique(gsub('-', '', sapply(as.character(dates),
  substr, 1, 7))))
# dates_file
#[1] "199405" "199406" "199407" "199408" "199409" "199410" "199411" "199412"
# -----------------------------------------------------------
  
# observational data
# May to December 1994, monthly file with 6-hourly frequency
  repos_obs <- paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
                      '$var$/$var$_$file_date$.nc')
  erai <- Start(dat = repos_obs,
                var = 'tas',
                file_date = dates_file,
                time = values(dates), 
                latitude = indices(1:10),
                longitude = indices(1:10),
                time_var = 'time',
                #because time is assigned by 'values', set the tolerance to avoid too distinct match
                time_tolerance = as.difftime(1, units = 'hours'), 
                #time values are across all the files
                time_across = 'file_date',  
                #combine time and file_date dims 
                merge_across_dims = TRUE,
                #exclude the additional NAs generated by merge_across_dims
                merge_across_dims_narm = TRUE,
                #split time dim, because it is two-dimensional
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = 'file_date'),
                retrieve = TRUE)

# NOTE: 'merge_across_dims_narm = TRUE' is necessary because the observational 
#       data have unequal time length of 30-day and 31-day months.
#       If the NAs are not removed, unwanted NAs will exist and make the
#       values misplaced in the array. See 'bonus' below for more explanation.

#------- Check erai -----------
dim(erai)
#      dat       var     sdate      time  latitude longitude 
#        1         1         8        31        10        10 

erai[1, 1, 1, , 1, 1]
# [1] 255.0120 256.8095 254.3654 254.6059 257.0551 255.5087 256.8167 257.9717
# [9] 258.7491 259.2942 259.6682 260.7215 260.0988 261.2605 263.3590 265.6683
#[17] 262.4813 262.6136 263.0591 262.8377 261.7276 263.9910 264.7755 266.0213
#[25] 268.5927 267.8699 268.9210 269.4702 267.6735 267.9255 268.2216

erai[1, 1, , 1, 2, 2]
#[1] 254.5793 269.6221 274.5021 274.0269 269.5855 253.7458 243.9750 244.2415

# NOTE: You will see that the observation array is the same as experiment one
#       that the last day in 30-day months are the first day of the following
#       month.
erai[1, 1, 3, 1, 1, 1]  # 1st March
#[1] 274.6019
erai[1, 1, 2, 31, 1, 1]  # 1st March also, since June only has 30 days
#[1] 274.6019
#------------------------------

# The experimental and observational data are comparable with same structure.


# //////////////////"BONUS"//////////////////////
# Here is something more to show the usage of parameter 'merge_across_dims_narm'.
# If the last day of 30-day months is NA instead of the first day of the following month,
# NAs are needed to exist in the array. In this case, 'merge_across_dims_narm'
# should be FALSE.

 dates <- attr(system4, 'Variables')$common$time
 dates[2, 31]
#[1] "1994-07-01 UTC"
 dates[2, 31] <- NA  # Jun
 dates[5, 31] <- NA  # Sep
 dates[7, 31] <- NA  # Nov

  erai <- Start(dat = repos_obs,
                var = 'tas',
                file_date = dates_file,
                time = values(dates),
                latitude = indices(1:10),
                longitude = indices(1:10),
                time_var = 'time',
                time_across = 'file_date',
                merge_across_dims = TRUE,
                #keep NAs of the last day in 30-day months
                merge_across_dims_narm = FALSE,
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = 'file_date'),
                retrieve = TRUE)

#------- Check erai -----------
erai[1, 1, 2, , 1, 1] # June
# [1] 269.9410 269.6855 268.7380 268.5008 270.3236 271.5151 270.5046 270.1686
# [9] 270.5395 272.0379 272.5489 271.1494 270.7764 270.5678 272.0331 273.7856
#[17] 273.9849 274.5904 273.4369 273.8404 274.4068 274.2292 274.7375 275.5104
#[25] 275.4324 274.9408 274.8679 276.5602 275.0995 274.6409       NA
erai[1, 1, 5, , 1, 1] # Sep
# [1] 270.0656 270.7113 268.4678 271.6489 271.2354 269.7831 269.8045 268.7994
# [9] 266.3092 262.2734 265.0124 261.8378 265.3950 257.1690 255.8402 264.8826
#[17] 267.8663 266.6875 262.5502 258.5476 258.9617 263.6396 257.1111 264.8644
#[25] 261.0085 256.7690 256.5811 256.4331 256.1260 256.4716       NA
#------------------------------
