context("Start() metadata_dims check")

test_that("1. One data set, one var", {
  repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
  data <- Start(dat = list(list(name = 'system5_m1', path = repos)),
                var = 'tas',
                sdate = '20170101',
                ensemble = indices(1),
                time = indices(1),
                lat = indices(1:10),
                lon = indices(1:10),
                synonims = list(lat = c('lat', 'latitude'),
                                lon = c('lon', 'longitude')),
                return_vars = list(time = 'sdate',
                                   lon = 'dat',
                                   lat = 'dat'),
                metadata_dims = 'dat',  # it can be omitted since it is automatically specified as 'dat'
                retrieve = T
               )

  expect_equal(
  length(attr(data, 'Variables')),
  2
  )
  expect_equal(
  names(attr(data, 'Variables')),
  c("common", "system5_m1")
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  c('time', 'tas')
  )
  expect_equal(
  names(attr(data, 'Variables')$system5_m1),
  c("lon", "lat")
  )
  expect_equal(
  length(attr(data, 'Variables')$common$tas),
  12
  )

})


test_that("2. Two data sets, one var", {
  repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
  repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"

  data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
                           list(name = 'system5_m1', path = repos)),
                var = 'tas',
                sdate = '20170101',
                ensemble = indices(1),
                time = indices(1),
                lat = indices(1:10),
                lon = indices(1:10),
                synonims = list(lat = c('lat', 'latitude'),
                                lon = c('lon', 'longitude')),
                return_vars = list(time = 'sdate',
                                   lon = 'dat',
                                   lat = 'dat'),
                metadata_dims = 'dat',  # it can be omitted since it is automatically specified as 'dat'
                retrieve = T
               )

  expect_equal(
  length(attr(data, 'Variables')),
  3
  )
  expect_equal(
  names(attr(data, 'Variables')),
  c("common", "system4_m1", "system5_m1")
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  'time'
  )
  expect_equal(
  names(attr(data, 'Variables')$system4_m1),
  c("lon", "lat", "tas")
  )
  expect_equal(
  names(attr(data, 'Variables')$system5_m1),
  c("lon", "lat", "tas")
  )
  expect_equal(
  length(attr(data, 'Variables')$system5_m1$tas),
  12
  )
  expect_equal(
  length(attr(data, 'Variables')$system4_m1$tas),
  11
  )

})

test_that("3. One data set, two vars", {
  repos <- paste0('/esarchive/exp/ecearth/a1tr/cmorfiles/CMIP/EC-Earth-Consortium/',
                  'EC-Earth3/historical/r24i1p1f1/Amon/$var$/gr/v20190312/',
                  '$var$_Amon_EC-Earth3_historical_r24i1p1f1_gr_185001-185012.nc')
  var <- c('tas', 'clt')
  sdate <- '20170101'

  data <- Start(dat = repos,
                var = var, 
                time = indices(1),
                lat = indices(1:10),
                lon = indices(10:19), 
                return_vars = list(lat = NULL, lon = NULL),
                metadata_dims = 'var',
                synonims = list(lat = c('lat', 'latitude'),
                                lon = c('lon', 'longitude')),
                retrieve = TRUE
               )

  expect_equal(
  length(attr(data, 'Variables')),
  2
  )
  expect_equal(
  names(attr(data, 'Variables')),
  c("common", "dat1")
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  c('lat', 'lon', 'tas', 'clt')
  )
  expect_equal(
  is.null(attr(data, 'Variables')$dat1),
  TRUE
  )
  expect_equal(
  length(attr(data, 'Variables')$common$tas),
  17
  )
  expect_equal(
  length(attr(data, 'Variables')$common$clt),
  16
  )

})

test_that("4. Two data sets, two vars", {
  repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
  repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"

  data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
                           list(name = 'system5_m1', path = repos)),
                var = c('tas', 'sfcWind'),
                sdate = '20170101',
                ensemble = indices(1),
                time = indices(1),
                lat = indices(1:10),
                lon = indices(1:10),
                synonims = list(lat = c('lat', 'latitude'),
                                lon = c('lon', 'longitude')),
                return_vars = list(time = 'sdate',
                                   lon = 'dat',
                                   lat = 'dat'),
                metadata_dims = 'dat',
                retrieve = T
               )

  expect_equal(
  length(attr(data, 'Variables')),
  3
  )
  expect_equal(
  names(attr(data, 'Variables')),
  c("common", "system4_m1", "system5_m1")
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  'time'
  )
  expect_equal(
  names(attr(data, 'Variables')$system4_m1),
  c("lon", "lat", "tas")
  )
  expect_equal(
  names(attr(data, 'Variables')$system5_m1),
  c("lon", "lat", "tas")
  )
  expect_equal(
  length(attr(data, 'Variables')$system5_m1$tas),
  12
  )
  expect_equal(
  length(attr(data, 'Variables')$system4_m1$tas),
  11
  )


  data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
                           list(name = 'system5_m1', path = repos)),
                var = c('tas', 'sfcWind'),
                sdate = '20170101',
                ensemble = indices(1),
                time = indices(1),
                lat = indices(1:10),
                lon = indices(1:10),
                synonims = list(lat = c('lat', 'latitude'),
                                lon = c('lon', 'longitude')),
                return_vars = list(time = 'sdate',
                                   lon = 'dat',
                                   lat = 'dat'),
                metadata_dims = c('dat', 'var'),
                retrieve = T
               )
  expect_equal(
  length(attr(data, 'Variables')),
  3
  )
  expect_equal(
  names(attr(data, 'Variables')),
  c("common", "system4_m1", "system5_m1")
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  'time'
  )
  expect_equal(
  names(attr(data, 'Variables')$system4_m1),
  c("lon", "lat", "tas", "sfcWind")
  )
  expect_equal(
  names(attr(data, 'Variables')$system5_m1),
  c("lon", "lat", "tas", "sfcWind")
  )
  expect_equal(
  length(attr(data, 'Variables')$system5_m1$tas),
  12
  )
  expect_equal(
  length(attr(data, 'Variables')$system5_m1$sfcWind),
  12
  )
  expect_equal(
  length(attr(data, 'Variables')$system4_m1$tas),
  11
  )
  expect_equal(
  length(attr(data, 'Variables')$system4_m1$sfcWind),
  11
  )

})

test_that("5. Specify metadata_dims with another file dimension", {
  repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
  repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"

  data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
                           list(name = 'system5_m1', path = repos)),
                var = c('tas', 'sfcWind'),
                sdate = c('20150101', '20160101', '20170101'),
                ensemble = indices(1),
                time = indices(1),
                lat = indices(1:10),
                lon = indices(1:10),
                synonims = list(lat = c('lat', 'latitude'),
                                lon = c('lon', 'longitude')),
                return_vars = list(time = 'sdate',
                                   lon = 'dat',
                                   lat = 'dat'),
                metadata_dims = 'sdate',
                retrieve = T
               )
  expect_equal(
  length(attr(data, 'Variables')),
  3
  )
  expect_equal(
  names(attr(data, 'Variables')),
  c("common", "system4_m1", "system5_m1")
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  c('time', 'tas', 'tas', 'tas')
  )
  expect_equal(
  names(attr(data, 'Variables')$system4_m1),
  c("lon", "lat")
  )
  expect_equal(
  names(attr(data, 'Variables')$system5_m1),
  c("lon", "lat")
  )
  expect_equal(
  length(attr(data, 'Variables')$common$tas),
  12
  )

})


test_that("6. One data set, two vars from one file", {
mask_path <- '/esarchive/autosubmit/con_files/mask.regions.Ec3.0_O1L46.nc'

data <- Start(repos = mask_path,
              var = c('nav_lon', 'nav_lat'),
              t = 'first',
              z = 'first',
              x = 'all',
              y = 'all',
              return_vars = list(var_names = NULL),
              var_var = 'var_names',
              retrieve = T)

  expect_equal(
  length(attr(data, 'Variables')),
  2
  )
  expect_equal(
  names(attr(data, 'Variables')),
  c("common", "dat1")
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  c("var_names", "nav_lon", "nav_lat")
  )
  expect_equal(
  is.null(attr(data, 'Variables')$dat1),
  TRUE
  )
  expect_equal(
  length(attr(data, 'Variables')$common$nav_lat),
  8
  )

})

test_that("7. Two data sets, while one is missing", {
 repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
 # incorrect path. Therefore repos2 doesn't have any valid files 
 repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f2h/$var$_$sdate$.nc"  # correct one is _f6h
  var <- 'tas'
  data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
                           list(name = 'system5_m1', path = repos)),
                var = var,
                sdate = '20170101',
                ensemble = indices(1),
                time = indices(1),
                lat = 'all',
                lon = 'all',
                synonims = list(lat = c('lat', 'latitude'),
                                lon = c('lon', 'longitude')),
                return_vars = list(time = 'sdate',
                                   lon = 'dat',
                                   lat = 'dat'),
                metadata_dims = 'dat',  # it can be omitted since it is automatically specified as 'dat'
                retrieve = T
               )

  expect_equal(
  length(data[is.na(data)]),
  829440
  )
  expect_equal(
  attr(data, "Variables")$system4_m1,
  NULL
  )
  expect_equal(
  length(attr(data, "Variables")$system5_m1$lon),
  1296
  )
  expect_equal(
  length(attr(data, "Variables")$system5_m1),
  3
  )
  expect_equal(
  attr(data, 'Files'),
  array(c(NA, "/esarchive/exp/ecmwf/system5_m1/monthly_mean/tas_f6h/tas_20170101.nc"),
        dim = c(dat = 2, var = 1, sdate = 1))
  )
})

test_that("8. Two data sets, both have files but the first file is missing", {
path_list <- list(
              MPI = list(name = 'MPI_ESM',
                         path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/',
                                       'cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/',
                                       'dcppA-hindcast/$member$/day/$var$/gn/v20200101/',
                                       '$var$_day_MPI-ESM1-2-HR_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc')),
              Had = list(name = 'HadGEM3',
                         path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/',
                                      'cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/',
                                      'dcppA-hindcast/$member$/day/$var$/gn/v20200101/',
                                      '$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc')))

data <- Start(dataset = path_list,
              var = 'tasmin',
              member = list(c('r1i1p1f1', 'r2i1p1f2')), 
              sdate = paste0(2018),
              chunk = list(c('20181101-20281231', '20181101-20281230')),
              time = indices(1), #'all',
              lat = values(list(0, 14)),
              lon = values(list(0, 28)),
              synonims = list(time = c('fmonth','time'),
                              lon = c('lon', 'longitude'),
                              lat = c('lat', 'latitude')),
              return_vars = list(lat = 'dataset', lon = 'dataset'),
              lat_reorder = Sort(),
              num_procs = 1,
              retrieve = T)

  expect_equal(
  length(data[is.na(data)]),
  5500
  )
  expect_equal(
  length(attr(data, "Variables")$MPI_ESM),
  3
  )
  expect_equal(
  length(attr(data, "Variables")$MPI_ESM$lon),
  30
  )
  expect_equal(
  names(attr(data, "Variables")$MPI_ESM),
  c('lat', 'lon', 'tasmin')
  )
  expect_equal(
  length(attr(data, "Variables")$HadGEM3),
  2
  )
  expect_equal(
  length(attr(data, "Variables")$HadGEM3$lon),
  34
  )
  expect_equal(
  names(attr(data, "Variables")$HadGEM3),
  c('lat', 'lon')
  )
  expect_equal(
  attr(data, 'Files'),
  array(c("/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/day/tasmin/gn/v20200101/tasmin_day_MPI-ESM1-2-HR_dcppA-hindcast_s2018-r1i1p1f1_gn_20181101-20281231.nc",
          NA, NA, NA, NA, NA, NA,
          "/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r2i1p1f2/day/tasmin/gn/v20200101/tasmin_day_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r2i1p1f2_gn_20181101-20281230.nc"),
        dim = c(dataset = 2, var = 1, member = 2, sdate = 1, chunk = 2))
  )


})
