# When certain inner dim of files is not consistent, the parameter 'largest_dims_length' can
# be used to ensure the returned array has the largest length of inner dimensions.

test_that("1. inconsistent member length", {
# system5c3s: sdate = '19931101' has 25 members, '20200901' has 51 members.
# system3: 40 members.

repos <- list(list(name = 'system5c3s',
                   path = "/esarchive/scratch/aho/startR_unittest_files/esarchive/exp/ecmwf/system5c3s/monthly_mean/g500_f12h/$var$_$sdate$.nc"),
              list(name = 'system3_m1-c3s',
                   path = "/esarchive/scratch/aho/startR_unittest_files/esarchive/exp/cmcc/system3_m1-c3s/monthly_mean/g500_f12h/$var$_$sdate$.nc"))


# largest_dims_length = FALSE
suppressWarnings(
dat1 <- Start(dataset = repos,
              var = "g500", 
              sdate = c("19931101","20200901"),
              time = 'all',
              ensemble = 'all',
              latitude = values(list(10, 12)),      
              latitude_reorder = Sort(decreasing = T),              
              longitude = values(list(10, 11)),              
              longitude_reorder = CircularSort(0, 360),
              largest_dims_length = FALSE,
              synonims = list(longitude = c('lon', 'longitude'),                              
                              latitude = c('lat', 'latitude')),              
              return_vars = list(     time = NULL,
                                      latitude = 'dataset',                            
                                      longitude = 'dataset'),          
              retrieve = T)
)
  expect_equal(
  dim(dat1),
  c(dataset = 2, var = 1, sdate = 2, time = 6, ensemble = 40, latitude = 3, longitude = 2)
  )
  expect_equal(
  length(dat1[is.na(dat1)]),
  3000
  )
  expect_equal(
  sum(as.numeric(is.na(dat1[1, 1, 1, , , , ]))),
  540
  )
  expect_equal(
  sum(as.numeric(is.na(dat1[1, 1, 2, , , , ]))),
  540
  )

# largest_dims_length = TRUE
suppressWarnings(
dat2 <- Start(dataset = repos,
              var = "g500",
              sdate = c("19931101","20200901"),
              time = 'all',
              ensemble = 'all',
              latitude = values(list(10, 12)),
              latitude_reorder = Sort(decreasing = T),
              longitude = values(list(10, 11)),
              longitude_reorder = CircularSort(0, 360),
              largest_dims_length = TRUE,
              synonims = list(longitude = c('lon', 'longitude'),
                              latitude = c('lat', 'latitude')),
              return_vars = list(     time = NULL,
                                      latitude = 'dataset',
                                      longitude = 'dataset'),
              retrieve = T)
)
  expect_equal(
  dim(dat2),
  c(dataset = 2, var = 1, sdate = 2, time = 6, ensemble = 51, latitude = 3, longitude = 2)
  )
  expect_equal(
  length(dat2[is.na(dat2)]),
  3528
  )
  expect_equal(
  sum(as.numeric(is.na(dat2[1, 1, 1, , , , ]))),
  936
  )
  expect_equal(
  sum(as.numeric(is.na(dat2[1, 1, 2, , , , ]))),
  0
  )

# largest_dims_length = c(ensemble = 51)
suppressWarnings(
dat3 <- Start(dataset = repos,
              var = "g500",
              sdate = c("19931101","20200901"),
              time = 'all',
              ensemble = 'all',
              latitude = values(list(10, 12)),
              latitude_reorder = Sort(decreasing = T),
              longitude = values(list(10, 11)),
              longitude_reorder = CircularSort(0, 360),
              largest_dims_length = c(ensemble = 51),
              synonims = list(longitude = c('lon', 'longitude'),
                              latitude = c('lat', 'latitude')),
              return_vars = list(     time = NULL,
                                      latitude = 'dataset',
                                      longitude = 'dataset'),
              retrieve = T)
)
  expect_equal(
  dim(dat3),
  c(dataset = 2, var = 1, sdate = 2, time = 6, ensemble = 51, latitude = 3, longitude = 2)
  )
  expect_equal(
  length(dat3[is.na(dat3)]),
  3528
  )
  expect_equal(
  sum(as.numeric(is.na(dat3[1, 1, 1, , , , ]))),
  936
  )
  expect_equal(
  sum(as.numeric(is.na(dat3[1, 1, 2, , , , ]))),
  0
  )

# Compare dat1, dat2, and dat3
  expect_equal(
  mean(dat2, na.rm = T),
  mean(dat3, na.rm = T)
  )
  expect_equal(
  mean(dat1[, , , , 1:25, , ], na.rm = T),
  mean(dat2[, , , , 1:25, , ], na.rm = T)
  )
  expect_equal(
  mean(dat1[, , , , 1:25, , ], na.rm = T),
  mean(dat3[, , , , 1:25, , ], na.rm = T)
  )

})

test_that("2. inconsistent time length, merge_across_dims = T", {

path <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/EC-Earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/',
               '$member$/Amon/$var$/gr/v20210309/',
               '$var$_Amon_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$.nc')
path <- paste0('/esarchive/scratch/aho/startR_unittest_files/', path)

suppressWarnings(
data <- Start(dataset = path,
              var = 'psl',
              sdate = '1970',
              lat = indices(1),
              lon = indices(1),
              fyear = c('197011-197012', '197101-197112'),  #2+12
              fmonth = 'all',
              member = 'r6i2p1f1',
              fmonth_across = 'fyear',
              merge_across_dims = TRUE,
              merge_across_dims_narm = TRUE,
              largest_dims_length = TRUE, #c(fmonth = 12),
              synonims = list(fmonth = c('time', 'fmonth'),
                              lon = c('lon', 'longitude'),
                              lat = c('lat', 'latitude')),
              return_vars = list(lat = 'dataset', lon = 'dataset',
                                 fmonth = 'fyear'),
              retrieve = TRUE)
)
expect_equal(
dim(data),
c(dataset = 1, var = 1, sdate = 1, lat = 1, lon = 1, fmonth = 14, member = 1)
)
expect_equal(
as.vector(data)[1:5],
c(101341.03, 100831.62, 99877.38, 101355.11, 101067.74),
tolerance = 0.001
)

suppressWarnings(
data2 <- Start(dataset = path,
              var = 'psl',
              sdate = '1970',
              lat = indices(1),
              lon = indices(1),
              fyear = c('197011-197012', '197101-197112'),  #2+12
              fmonth = indices(1:14),
              member = 'r6i2p1f1',
              fmonth_across = 'fyear',
              merge_across_dims = TRUE,
              merge_across_dims_narm = TRUE,
              largest_dims_length = TRUE, #c(fmonth = 12),
              synonims = list(fmonth = c('time', 'fmonth'),
                              lon = c('lon', 'longitude'),
                              lat = c('lat', 'latitude')),
              return_vars = list(lat = 'dataset', lon = 'dataset',
                                 fmonth = 'fyear'),
              retrieve = TRUE)
)

expect_equal(
as.vector(data),
as.vector(data2)
)

suppressWarnings(
data3 <- Start(dataset = path,
              var = 'psl',
              sdate = '1970',
              lat = indices(1),
              lon = indices(1),
              fyear = c('197011-197012', '197101-197112'),  #2+12
              fmonth = 'all',
              member = 'r6i2p1f1',
              fmonth_across = 'fyear',
              merge_across_dims = TRUE,
              merge_across_dims_narm = TRUE,
              largest_dims_length = c(fmonth = 12),
              synonims = list(fmonth = c('time', 'fmonth'),
                              lon = c('lon', 'longitude'),
                              lat = c('lat', 'latitude')),
              return_vars = list(lat = 'dataset', lon = 'dataset',
                                 fmonth = 'fyear'),
              retrieve = TRUE)
)

expect_equal(
as.vector(data),
as.vector(data3)
)

#--------------------------------
suppressWarnings(
data4 <- Start(dataset = path,
              var = 'psl',
              sdate = c('1970', '1971'),
              lat = indices(1),
              lon = indices(1),
              fyear = 'all',
              fmonth = 'all',
              fyear_depends = 'sdate',
              member = 'r6i2p1f1',
              fmonth_across = 'fyear',
              merge_across_dims = TRUE,
              merge_across_dims_narm = TRUE,
              largest_dims_length = TRUE,
              synonims = list(fmonth = c('time', 'fmonth'),
                              lon = c('lon', 'longitude'),
                              lat = c('lat', 'latitude')),
              return_vars = list(lat = 'dataset', lon = 'dataset',
                                 fmonth = 'fyear'),
              retrieve = TRUE)
)

expect_equal(
dim(data4),
c(dataset = 1, var = 1, sdate = 2, lat = 1, lon = 1, fmonth = 122, member = 1)
)
expect_equal(
as.vector(drop(data4)[1,1:5]),
c(101341.03, 100831.62, 99877.38, 101355.11, 101067.74),
tolerance = 0.001
)
expect_equal(
as.vector(drop(data4)[2,10:14]),
c(100515.2, 101354.9, 100657.0, 100349.4, 100199.6),
tolerance = 0.001
)

#--------------------------------
suppressWarnings(
data5 <- Start(dataset = path,
              var = 'psl',
              sdate = c('1970', '1971'),
              lat = indices(1),
              lon = indices(1),
              fyear = 'all',
              fmonth = 'all',
              fyear_depends = 'sdate',
              member = 'r6i2p1f1',
              fmonth_across = 'fyear',
              merge_across_dims = TRUE,
              merge_across_dims_narm = FALSE, #TRUE,
              largest_dims_length = TRUE,
              synonims = list(fmonth = c('time', 'fmonth'),
                              lon = c('lon', 'longitude'),
                              lat = c('lat', 'latitude')),
              return_vars = list(lat = 'dataset', lon = 'dataset',
                                 fmonth = 'fyear'),
              retrieve = TRUE)
)

expect_equal(
dim(data5),
c(dataset = 1, var = 1, sdate = 2, lat = 1, lon = 1, fmonth = 132, member = 1)
)
expect_equal(
which(is.na(drop(data5))),
c(5:24)
)
expect_equal(
as.vector(data4),
as.vector(data5)[-c(5:24)]
)

})
