context("Start() reshape parameters check")
# This one is more comprehensive than test-Start-split-merge.R

path_exp <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/$var$_f6h/$var$_$sdate$.nc'
path_obs <- '/esarchive/recon/ecmwf/era5/daily_mean/$var$_f1h-r360x181/$var$_$date$.nc'
var <- 'tas'
sdate <- paste0(1993:1995, '1201')

suppressWarnings(
exp <- Start(dat = path_exp,
             var = var,
             sdate = sdate,
             time = indices(1:90), #indices(1:91),
             ensemble = indices(1),
             lat = indices(1),
             lon = indices(1),
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'sdate'),
             retrieve = FALSE, silent = T)
)
dates <- attr(exp, 'Variables')$common$time

# easyNCDF
library(easyNCDF)
# obs
easy_sdate <- c('199312', paste0(rep(1994:1995, each = 3), c('01', '02', '12')), 
                '199601', '199602')
easy_array <- c()

for (i in 1:length(easy_sdate)) {
  easy_file <- NcOpen(paste0('/esarchive/recon/ecmwf/era5/daily_mean/tas_f1h-r360x181/tas_',
                             easy_sdate[i], '.nc'))
  if (substr(easy_sdate[i], 5, 6) == '02') {
    sub_time <- 1:28
  } else {
    sub_time <- 1:31
  }
  easy_obs <- NcToArray(easy_file, vars_to_read = 'tas',
                        dim_indices = list(lon = c(1), lat = c(1), time = sub_time))
  NcClose(easy_file)
  easy_array <- c(easy_array, as.vector(easy_obs))
}
dim(easy_array) <- c(time = 90, sdate = 3)



test_that("1. split + merge + narm", {

sorted_dates <- sort(unique(format(dates, '%Y%m')))
unsorted_dates <- unique(format(dates, '%Y%m'))

# unsorted dates
obs1 <- Start(dat = path_obs,
             var = var,
             date = unsorted_dates,
             time = values(dates),  #dim: [sdate = 3, time = 90]
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
             merge_across_dims_narm = TRUE,
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)

# sorted_dates
obs2 <- Start(dat = path_obs,
             var = var,
             date = sorted_dates,
             time = values(dates),  #dim: [sdate = 3, time = 90]
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
             merge_across_dims_narm = TRUE,
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)

expect_equal(
dim(obs1),
c(dat = 1, var = 1, sdate = 3, time = 90, lat = 1, lon = 1)
)
expect_equal(
dim(obs1),
dim(obs2)
)
expect_equal(
as.vector(obs1),
as.vector(obs2)
)
expect_equal(
as.vector(obs1[1, 1, 1, , 1, 1]),
as.vector(easy_array[, 1])
)
expect_equal(
as.vector(obs1[1, 1, 2, , 1, 1]),
as.vector(easy_array[, 2])
)
expect_equal(
as.vector(obs1[1, 1, 3, , 1, 1]),
as.vector(easy_array[, 3])
)

})


test_that("2. split + merge", {

exp <- Start(dat = path_exp,
             var = var,
             sdate = sdate,
             time = indices(1:62),
             ensemble = indices(1),
             lat = indices(1),
             lon = indices(1),
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'sdate'),
             retrieve = FALSE)

dates <- attr(exp, 'Variables')$common$time

sorted_dates <- sort(unique(format(dates, '%Y%m')))
unsorted_dates <- unique(format(dates, '%Y%m'))

# unsorted dates
obs1 <- Start(dat = path_obs,
             var = var,
             date = unsorted_dates,
             time = values(dates),  #dim: [sdate = 3, time = 62]
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
#             merge_across_dims_narm = TRUE,
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)

# sorted_dates
obs2 <- Start(dat = path_obs,
             var = var,
             date = sorted_dates,
             time = values(dates),  #dim: [sdate = 3, time = 62]
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
#             merge_across_dims_narm = TRUE,
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)

expect_equal(
dim(obs1),
c(dat = 1, var = 1, sdate = 3, time = 62, lat = 1, lon = 1)
)
expect_equal(
as.vector(obs1[1, 1, 1, , 1, 1]),
as.vector(easy_array[1:62, 1])
)
expect_equal(
as.vector(obs1[1, 1, 2, , 1, 1]),
as.vector(easy_array[1:62, 2])
)
expect_equal(
as.vector(obs1[1, 1, 3, , 1, 1]),
as.vector(easy_array[1:62, 3])
)
expect_equal(
as.vector(obs1),
as.vector(obs2)
)

})



test_that("3. merge", {
# NOTE: The three files are all regarded to have time = 31, despite 199402 only has 28.
#       It happens when time = 'all' or time = indices(). It seems reasonable when 
#       'merge_across_dims' is not used, but if it is used, it's common to expect 31+31+28.
#       See the next test "4. merge + narm". 199402 is still regarded as 31, so NAs are not
#       removed.
suppressWarnings(
obs3 <- Start(dat = path_obs,
             var = var,
             date = c('199312', '199401', '199402'),
             time = 'all',
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
#             merge_across_dims_narm = TRUE,
#             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)
)


expect_equal(
dim(obs3),
c(dat = 1, var = 1, time = 93, lat = 1, lon = 1)
)
expect_equal(
as.vector(obs3),
c(as.vector(easy_array[, 1]), NA, NA, NA)
)

})


test_that("4. merge + narm", {

# (1) Notice that the NAs at the tail of 199402 won't be removed because Start()
#     considers all the files have the same length, i.e., 31.
#     The NAs in 199402 are regarded as part of the original file.

obs3 <- Start(dat = path_obs,
             var = var,
             date = c('199312', '199401', '199402'),
             time = 'all',
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
             merge_across_dims_narm = TRUE,
#             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)

expect_equal(
dim(obs3),
c(dat = 1, var = 1, time = 93, lat = 1, lon = 1)
)
expect_equal(
as.vector(obs3),
c(as.vector(easy_array[, 1]), NA, NA, NA)
)

# (2) It's tricky that 199402 is considered time = 31 because Start() considers
#     all the files have the same length. So it won't return an error when 
#     time = indices(93).
#     The first 14 time steps of 199312 will be removed but the NAs at the tail
#     of 199402 will be preserved.
obs4 <- Start(dat = path_obs,
             var = var,
             date = c('199312', '199401', '199402'),
             time = indices(15:93),
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
             merge_across_dims_narm = TRUE,
#             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)

expect_equal(
dim(obs4),
c(dat = 1, var = 1, time = 79, lat = 1, lon = 1)
)
expect_equal(
as.vector(obs4),
c(as.vector(easy_array[15:90, 1]), NA, NA, NA)
)

# (3) If time is values(), 199402 is considered time = 28, so NAs will be removed.
obs5 <- Start(dat = path_obs,
             var = var,
             date = c('199312', '199401', '199402'),
             time = dates[1, ],
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
             merge_across_dims_narm = TRUE,
#             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)
expect_equal(
dim(obs5),
c(dat = 1, var = 1, time = 90, lat = 1, lon = 1)
)
expect_equal(
as.vector(obs5),
as.vector(easy_array[1:90, 1])
)

})

test_that("5. split", {

date_array <- c('199312', '199401', '199412', '199501')
dim(date_array) <- c(month = 2, year = 2)

# split file dim
obs1 <- Start(dat = path_obs,
             var = var,
             date = date_array, # [month = 2, year = 2]
             time = indices(1:31),
             lat = indices(1),
             lon = indices(1),
#             time_across = 'date',
#             merge_across_dims = TRUE,
#             merge_across_dims_narm = TRUE,
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL),
                              #  time = 'date'),
             retrieve = TRUE)

expect_equal(
dim(obs1),
c(dat = 1, var = 1, month = 2, year = 2, time = 31, lat = 1, lon = 1)
)
expect_equal(
as.vector(obs1[1, 1, 1, 1, , 1, 1]),
as.vector(easy_array[1:31, 1])
)
expect_equal(
as.vector(obs1[1, 1, 2, 1, , 1, 1]),
as.vector(easy_array[32:62, 1])
)
expect_equal(
as.vector(obs1[1, 1, 1, 2, , 1, 1]),
as.vector(easy_array[1:31, 2])
)
expect_equal(
as.vector(obs1[1, 1, 2, 2, , 1, 1]),
as.vector(easy_array[32:62, 2])
)

# split inner time
## time is indices
time_array <- array(1:62, dim = c(day = 31, month = 2))
exp1 <- Start(dat = path_exp,
             var = var,
             sdate = sdate[1],
             time = time_array,
             ensemble = indices(1),
             lat = indices(1),
             lon = indices(1),
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'sdate'),
             retrieve = TRUE)

# easyNCDF
easy_sdate_exp <- '19931201'
easy_file_exp <- NcOpen(paste0('/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_',
                               easy_sdate_exp, '.nc'))
easy_exp <- NcToArray(easy_file_exp, vars_to_read = 'tas',
                        dim_indices = list(longitude = c(1), latitude = c(1), ensemble = c(1),
                                           time = 1:62))
NcClose(easy_file_exp)

expect_equal(
dim(exp1),
c(dat = 1, var = 1, sdate = 1, day = 31, month = 2, ensemble = 1, lat = 1, lon = 1)
)
expect_equal(
as.vector(exp1),
as.vector(easy_exp)
)

## time is values
time_array <- dates[1, 1:62]
dim(time_array) <- c(day = 31, month = 2)
exp2 <- Start(dat = path_exp,
             var = var,
             sdate = sdate[1],
             time = time_array,
             ensemble = indices(1),
             lat = indices(1),
             lon = indices(1),
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL),
#                                time = 'sdate'),
             retrieve = TRUE)
expect_equal(
dim(exp2),
c(dat = 1, var = 1, sdate = 1, day = 31, month = 2, ensemble = 1, lat = 1, lon = 1)
)
expect_equal(
as.vector(exp1),
as.vector(exp2)
)


})

test_that("6. repetitive values", {

exp <- Start(dat = path_exp,
             var = var,
             sdate = c('19931101', '19931201'),
             time = indices(1:61),
             ensemble = indices(1),
             lat = indices(1),
             lon = indices(1),
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'sdate'),
             retrieve = F)
dates <- attr(exp, 'Variables')$common$time

# sorted and unsorted are the same here
sorted_dates <- sort(unique(format(dates, '%Y%m')))
#unsorted_dates <- unique(format(dates, '%Y%m'))

# sorted_dates
obs2 <- Start(dat = path_obs,
             var = var,
             date = sorted_dates,
             time = values(dates),  #dim: [sdate = 2, time = 61]
             lat = indices(1),
             lon = indices(1),
             time_across = 'date',
             merge_across_dims = TRUE,
             merge_across_dims_narm = TRUE,
             split_multiselected_dims = TRUE,
             synonims = list(lat = c('lat', 'latitude'),
                             lon = c('lon', 'longitude')),
             return_vars = list(lon = NULL,
                                lat = NULL,
                                time = 'date'),
             retrieve = TRUE)

# easyNCDF
easy_file_199311 <- NcOpen(paste0('/esarchive/recon/ecmwf/era5/daily_mean/tas_f1h-r360x181/tas_',
                                  '199311', '.nc'))
easy_obs_199311 <- NcToArray(easy_file_199311, vars_to_read = 'tas',
                             dim_indices = list(lon = c(1), lat = c(1), time = 1:30))
NcClose(easy_file_199311)

expect_equal(
dim(obs2),
c(dat = 1, var = 1, sdate = 2, time = 61, lat = 1, lon = 1)
)
expect_equal(
as.vector(obs2[1, 1, 1, 1:30, 1, 1]),
as.vector(easy_obs_199311)
)
expect_equal(
as.vector(obs2[1, 1, 1, 31:61, 1, 1]),
as.vector(obs2[1, 1, 2, 1:31, 1, 1])
)
expect_equal(
as.vector(obs2[1, 1, 2, 31:61, 1, 1]),
easy_array[31:61 ,1]
)

})
