context("Start() metadata reshaping")
# When data is reshaping (e.g., time_across = 'sdate'), the corresponding attribute should be reshaped too.

test_that("1. time across fyear, fyear depends on sdate", {

suppressWarnings(
data <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960:1961), 
                      time = 62:426, ## Jan to Dec (initialised in Nov)
                      time_across = 'fyear',
                      merge_across_dims = TRUE,
                      fyear = 'all',
                      fyear_depends = 'sdate', 
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         time = c('sdate', 'fyear')),
                      retrieve = TRUE)
)
dates <- attr(data,'Variables')$common[['time']]

expect_equal(
dim(dates),
c(sdate = 2, time = 365)
)
expect_equal(
dim(drop(data)),
dim(dates)
)
expect_equal(
length(attributes(dates)),
4
)
expect_equal(
all(names(attributes(dates)) %in% c('variables', 'dim', 'class', 'tzone')),
TRUE
)
expect_equal(
class(dates),
c("POSIXct", "POSIXt")
)
expect_equal(
as.vector(dates[1, ]),
as.vector(seq(as.POSIXct('1961-01-01 12:00:00', tz = 'UTC'), as.POSIXct('1961-12-31 12:00:00', tz = 'UTC'), by = 'day'))
)
expect_equal(
as.vector(dates[2, ]),
as.vector(seq(as.POSIXct('1962-01-01 12:00:00', tz = 'UTC'), as.POSIXct('1962-12-31 12:00:00', tz = 'UTC'), by = 'day'))
)

# retrieve = FALSE
suppressWarnings(
dataF <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960:1961),
                      time = 62:426, ## Jan to Dec (initialised in Nov)
                      time_across = 'fyear',
                      merge_across_dims = TRUE,
                      fyear = 'all',
                      fyear_depends = 'sdate',
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         time = c('sdate', 'fyear')),
                      retrieve = FALSE)
)
datesF <- attr(dataF,'Variables')$common[['time']]
expect_equal(
datesF,
dates
)

})


test_that("2. time across fyear, only one sdate", {

suppressWarnings(
data <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960),
                      time = 62:426, ## Jan to Dec (initialised in Nov)
                      time_across = 'fyear',
                      merge_across_dims = TRUE,
                      fyear = 'all',
#                      fyear_depends = 'sdate',
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         time = c('fyear')),
                      retrieve = TRUE)
)
dates <- attr(data,'Variables')$common[['time']]


expect_equal(
dim(dates),
c(time = 365)
)
expect_equal(
length(data),
length(dates)
)
expect_equal(
as.vector(dates),
as.vector(seq(as.POSIXct('1961-01-01 12:00:00', tz = 'UTC'), as.POSIXct('1961-12-31 12:00:00', tz = 'UTC'), by = 'day'))
)

#retrieve = FALSE
suppressWarnings(
dataF <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960),
                      time = 62:426, ## Jan to Dec (initialised in Nov)
                      time_across = 'fyear',
                      merge_across_dims = TRUE,
                      fyear = 'all',
#                      fyear_depends = 'sdate',
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         time = c('fyear')),
                      retrieve = FALSE)
)
datesF <- attr(dataF,'Variables')$common[['time']]
expect_equal(
datesF,
dates
)

})

test_that("3. time across fyear, fyear depends on sdate, 1st fyear is empty, 3rd fyear has more indices than 2nd one, 1964 is leap year", {

suppressWarnings(
data <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960:1961),
                      day = 700:860, ## initialised in Nov
                      day_across = 'fyear',
                      merge_across_dims = TRUE,
                      fyear = 'all',
                      fyear_depends = 'sdate',
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude'),
                                      day = c('day', 'time')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         day = c('sdate', 'fyear')),
                      retrieve = TRUE)
)
dates <- attr(data,'Variables')$common[['day']]

expect_equal(
dim(dates),
c(sdate = 2, day = 161)
)
expect_equal(
length(attributes(dates)),
4
)
expect_equal(
all(names(attributes(dates)) %in% c('variables', 'dim', 'class', 'tzone')),
TRUE
)
expect_equal(
class(dates),
c("POSIXct", "POSIXt")
)
expect_equal(
as.vector(dates[1, ]),
as.vector(seq(as.POSIXct('1962-10-01 12:00:00', tz = 'UTC'), as.POSIXct('1963-03-10 12:00:00', tz = 'UTC'), by = 'day'))
)
expect_equal(
as.vector(dates[2, ]),
as.vector(seq(as.POSIXct('1963-10-01 12:00:00', tz = 'UTC'), as.POSIXct('1964-03-09 12:00:00', tz = 'UTC'), by = 'day'))
)


suppressWarnings(
dataF <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960:1961),
                      day = 700:860, ## initialised in Nov
                      day_across = 'fyear',
                      merge_across_dims = TRUE,
                      fyear = 'all',
                      fyear_depends = 'sdate',
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude'),
                                      day = c('day', 'time')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         day = c('sdate', 'fyear')),
                      retrieve = FALSE)
)
datesF <- attr(dataF,'Variables')$common[['day']]
expect_equal(
datesF,
dates
)

})


test_that("4. merge and split time dim", {
datess <- seq(as.POSIXct('1994-05-01', tz = 'UTC'), as.POSIXct('1994-12-31', tz = 'UTC'), by = 'days') 
datess <- c(datess[c(1:31, 32:62, 62:92, 93:123, 124:154, 154:184, 185:215, 215:245)])
datess <- as.POSIXct(array(datess, dim = c(time = 31, sdate = 8)), 
                    origin = '1970-01-01', tz = 'UTC')
dates_file <- sort(unique(gsub('-', '', sapply(as.character(datess),
                   substr, 1, 7))))

suppressWarnings(
  data <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
                             '$var$/$var$_$file_date$.nc'),
                var = 'tas',
                file_date = dates_file,
                time = values(datess), #[time = 31, sdate = 8]
                latitude = indices(1),
                longitude = indices(1),
                time_var = 'time',
                #because time is assigned by 'values', set the tolerance to avoid too distinct match
                time_tolerance = as.difftime(1, units = 'hours'), 
                #time values are across all the files
                time_across = 'file_date',  
                #combine time and file_date dims 
                merge_across_dims = TRUE,
                #exclude the additional NAs generated by merge_across_dims
                merge_across_dims_narm = TRUE,
                #split time dim, because it is two-dimensional
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = 'file_date'),
                retrieve = TRUE)
)

dates <- attr(data,'Variables')$common[['time']]

expect_equal(
dim(dates),
c(time = 31, sdate = 8)
)
expect_equal(
length(attributes(dates)),
4
)
expect_equal(
all(names(attributes(dates)) %in% c('variables', 'dim', 'class', 'tzone')),
TRUE
)
expect_equal(
class(dates),
c("POSIXct", "POSIXt")
)
expect_equal(
as.vector(dates),
as.vector(datess)
)

suppressWarnings(
  dataF <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
                             '$var$/$var$_$file_date$.nc'),
                var = 'tas',
                file_date = dates_file,
                time = values(datess), #[time = 31, sdate = 8]
                latitude = indices(1),
                longitude = indices(1),
                time_var = 'time',
                #because time is assigned by 'values', set the tolerance to avoid too distinct match
                time_tolerance = as.difftime(1, units = 'hours'),
                #time values are across all the files
                time_across = 'file_date',
                #combine time and file_date dims 
                merge_across_dims = TRUE,
                #exclude the additional NAs generated by merge_across_dims
                merge_across_dims_narm = TRUE,
                #split time dim, because it is two-dimensional
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = 'file_date'),
                retrieve = FALSE)
)
datesF <- attr(dataF,'Variables')$common[['time']]
expect_equal(
datesF,
dates
)

})

test_that("5. test 1 but merge_across_dims_narm = F", {

suppressWarnings(
data <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960:1961),
                      time = 62:426, ## Jan to Dec (initialised in Nov)
                      time_across = 'fyear',
                      merge_across_dims = TRUE,
                      merge_across_dims_narm = F,
                      fyear = 'all',
                      fyear_depends = 'sdate',
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         time = c('sdate', 'fyear')),
                      retrieve = TRUE)
)
dates <- attr(data,'Variables')$common[['time']]

expect_equal(
dim(dates),
c(sdate = 2, time = 608)
)
expect_equal(
dim(drop(data)),
dim(dates)
)
expect_equal(
length(attributes(dates)),
4
)
expect_equal(
all(names(attributes(dates)) %in% c('variables', 'dim', 'class', 'tzone')),
TRUE
)
expect_equal(
class(dates),
c("POSIXct", "POSIXt")
)
expect_equal(
as.vector(dates[1, ]),
c(as.vector(seq(as.POSIXct('1961-01-01 12:00:00', tz = 'UTC'), as.POSIXct('1961-12-31 12:00:00', tz = 'UTC'), by = 'day')), rep(NA, 243))
)
expect_equal(
as.vector(dates[2, ]),
c(as.vector(seq(as.POSIXct('1962-01-01 12:00:00', tz = 'UTC'), as.POSIXct('1962-12-31 12:00:00', tz = 'UTC'), by = 'day')), rep(NA, 243))
)

suppressWarnings(
dataF <- Start(dat = '/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/$member$/day/$var$/gr/v20210910/$var$_day_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc',
                      var = 'tasmin',
                      lat = indices(1),
                      lon = indices(1),
                      sdate = paste0(1960:1961),
                      time = 62:426, ## Jan to Dec (initialised in Nov)
                      time_across = 'fyear',
                      merge_across_dims = TRUE,
                      merge_across_dims_narm = F,
                      fyear = 'all',
                      fyear_depends = 'sdate',
                      member = 'r1i4p1f1',
                      synonims = list(lat = c('lat','latitude'),
                                      lon = c('lon','longitude')),
                      return_vars = list(lat = NULL, lon = NULL,
                                         time = c('sdate', 'fyear')),
                      retrieve = FALSE)
)
datesF <- attr(dataF,'Variables')$common[['time']]
expect_equal(
datesF,
dates
)

})

test_that("6. split dim only", {

datess <- seq(as.POSIXct('1994-07-01', tz = 'UTC'), as.POSIXct('1994-07-14', tz = 'UTC'), by = 'days')
datess <- as.POSIXct(array(datess, dim = c(time = 7, week = 2)),
                     origin = '1970-01-01', tz = 'UTC')
suppressWarnings(
data <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
                             '$var$/$var$_199407.nc'),
                var = 'tas',
#                file_date = '199407',
                time = values(datess), #[time = 7, week = 2]
                latitude = indices(1),
                longitude = indices(1),
                time_var = 'time',
                time_tolerance = as.difftime(1, units = 'hours'),
#                time_across = 'file_date',
#                merge_across_dims = TRUE,
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = NULL), #'file_date'),
                retrieve = TRUE)
)
dates <- attr(data,'Variables')$common[['time']]


expect_equal(
dim(dates),
c(time = 7, week = 2)
)
expect_equal(
dim(drop(data)),
dim(dates)
)
expect_equal(
length(attributes(dates)),
4
)
expect_equal(
all(names(attributes(dates)) %in% c('variables', 'dim', 'class', 'tzone')),
TRUE
)
expect_equal(
class(dates),
c("POSIXct", "POSIXt")
)
expect_equal(
as.vector(dates[, ]),
as.vector(seq(as.POSIXct('1994-07-01', tz = 'UTC'), as.POSIXct('1994-07-14', tz = 'UTC'), by = 'day'))
)

suppressWarnings(
dataF <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
                             '$var$/$var$_199407.nc'),
                var = 'tas',
#                file_date = '199407',
                time = values(datess), #[time = 7, week = 2]
                latitude = indices(1),
                longitude = indices(1),
                time_var = 'time',
                time_tolerance = as.difftime(1, units = 'hours'),
#                time_across = 'file_date',
#                merge_across_dims = TRUE,
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = NULL), #'file_date'),
                retrieve = FALSE)
)
datesF <- attr(dataF,'Variables')$common[['time']]
expect_equal(
datesF,
dates
)

})

test_that("7. split dim + merge + merge_narm = F", {

datess <- seq(as.POSIXct('1994-07-01', tz = 'UTC'), as.POSIXct('1994-08-31', tz = 'UTC'), by = 'days')
datess <- as.POSIXct(array(datess, dim = c(time = 31, month = 2)),
                     origin = '1970-01-01', tz = 'UTC')

suppressWarnings(
data <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
                             '$var$/$var$_$file_date$.nc'),
                var = 'tas',
                file_date = c('199407', '199408'),
                time = values(datess), #[time = 31, month = 2]
                latitude = indices(1),
                longitude = indices(1),
                time_var = 'time',
                time_tolerance = as.difftime(1, units = 'hours'),
                time_across = 'file_date',
                merge_across_dims = TRUE,
                merge_across_dims_narm = F,
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = 'file_date'),
                retrieve = TRUE)
)
dates <- attr(data,'Variables')$common[['time']]


expect_equal(
dim(dates),
c(time = 31, month = 2)
)
expect_equal(
dim(drop(data)),
dim(dates)
)
expect_equal(
length(attributes(dates)),
4
)
expect_equal(
all(names(attributes(dates)) %in% c('variables', 'dim', 'class', 'tzone')),
TRUE
)
expect_equal(
class(dates),
c("POSIXct", "POSIXt")
)
expect_equal(
as.vector(dates[, ]),
as.vector(seq(as.POSIXct('1994-07-01', tz = 'UTC'), as.POSIXct('1994-08-31', tz = 'UTC'), by = 'day'))
)

suppressWarnings(
dataF <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
                             '$var$/$var$_$file_date$.nc'),
                var = 'tas',
                file_date = c('199407', '199408'),
                time = values(datess), #[time = 31, month = 2]
                latitude = indices(1),
                longitude = indices(1),
                time_var = 'time',
                time_tolerance = as.difftime(1, units = 'hours'),
                time_across = 'file_date',
                merge_across_dims = TRUE,
                merge_across_dims_narm = F,
                split_multiselected_dims = TRUE,
                return_vars = list(latitude = NULL,
                                   longitude = NULL,
                                   time = 'file_date'),
                retrieve = FALSE)
)
datesF <- attr(dataF,'Variables')$common[['time']]
expect_equal(
datesF,
dates
)

})