context("Start() retrieves files that the first file is missing")

# When some of the files are missing, Start() still can retrieve the data and 
# put NA in those missing positions. However, when the first file is missing,
# Start() returned error before because of failing to find metadata. The bug is
# fixed now, so even the first file does not exist, Start() will still retrieve
# the data. The parameter 'metadata_dims' can also be used in this case.

file <- "/esarchive/exp/ncep/cfs-v2/weekly_mean/s2s/$var$_f24h/$var$_$file_date$.nc"
var <- 'tas'
sdates1 <- c('20130611')  #exists
sdates2 <- c('20130618')  #does not exist
sdates3 <- c("20130611", "20130618")  #1st exists, 2nd missing
sdates4 <- c("20130618", "20130611")  #1st missing, 2nd exists
sdates5 <- c("20130611", "20130612")  #both exist

test_that("1. first file missing, no assign parameter 'metadata_dims'", {

suppressWarnings(
data <- Start(dat = file,
              var = var,
              file_date = sdates4,
              time = indices(1:4),
              latitude = values(list(20, 30)),
              latitude_reorder = Sort(decreasing = TRUE),
              longitude = values(list(-20, -10)),
              longitude_reorder = CircularSort(-180, 180),
              ensemble = indices(1),
              synonims = list(latitude = c('lat', 'latitude'),
                              longitude = c('lon', 'longitude')),
              return_vars = list(latitude = 'dat',
                                 longitude = 'dat',
                                 time = 'file_date'),
             #metadata_dims = c('file_date'),
              retrieve = T)
)

  expect_equal(
  dim(data),
  c(dat = 1, var = 1, file_date = 2, time = 4, latitude = 11, longitude = 11, ensemble = 1)
  )
  expect_equal(
  data[1, 1, , 1, 1, 1, 1],
  c(NA, 293.9133),
  tolerance = 0.0001
  )
  expect_equal(
  length(data[is.na(data)]),
  484
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  'time'
  )

})

test_that("2. Use parameter 'metadata_dims'", {
suppressWarnings(
data <- Start(dat = file,
              var = var,
              file_date = sdates4,
              time = indices(1:4),
              latitude = values(list(20, 30)),
              latitude_reorder = Sort(decreasing = TRUE),
              longitude = values(list(-20, -10)),
              longitude_reorder = CircularSort(-180, 180),
              ensemble = indices(1),
              synonims = list(latitude = c('lat', 'latitude'),
                              longitude = c('lon', 'longitude')),
              return_vars = list(latitude = 'dat',
                                 longitude = 'dat',
                                 time = 'file_date'),
              metadata_dims = c('file_date'),
              retrieve = T)
)

  expect_equal(
  dim(data),
  c(dat = 1, var = 1, file_date = 2, time = 4, latitude = 11, longitude = 11, ensemble = 1)
  )
  expect_equal(
  data[1, 1, , 1, 1, 1, 1],
  c(NA, 293.9133),
  tolerance = 0.0001
  )
  expect_equal(
  length(data[is.na(data)]),
  484
  )
  expect_equal(
  names(attr(data, 'Variables')$common),
  c('time', NA, 'tas')
  )
})

test_that("3. Use parameter 'metadata_dims', all common attributes, 1st file missing", {
suppressWarnings(
data <- Start(dat = file,
              var = var,
              file_date = sdates4,
              time = indices(1:4),
              latitude = values(list(20, 30)),
              latitude_reorder = Sort(decreasing = TRUE),
              longitude = values(list(-20, -10)),
              longitude_reorder = CircularSort(-180, 180),
              ensemble = indices(1),
              synonims = list(latitude = c('lat', 'latitude'),
                              longitude = c('lon', 'longitude')),
              return_vars = list(latitude = NULL,
                                 longitude = NULL,
                                 time = 'file_date'),
              metadata_dims = c('file_date'),
              retrieve = T)
)
  expect_equal(
  names(attr(data, 'Variables')$common),
  c('latitude', 'longitude', 'time', NA, 'tas')
  )
  expect_equal(
  as.vector(attr(data, 'NotFoundFiles')),
  c("/esarchive/exp/ncep/cfs-v2/weekly_mean/s2s/tas_f24h/tas_20130618.nc", NA)
  )

})

test_that("4. Use parameter 'metadata_dims', all common attributes, 2nd file missing", {
suppressWarnings(
data <- Start(dat = file,
              var = var,
              file_date = sdates3,
              time = indices(1:4),
              latitude = values(list(20, 30)),
              latitude_reorder = Sort(decreasing = TRUE),
              longitude = values(list(-20, -10)),
              longitude_reorder = CircularSort(-180, 180),
              ensemble = indices(1),
              synonims = list(latitude = c('lat', 'latitude'),
                              longitude = c('lon', 'longitude')),
              return_vars = list(latitude = NULL,
                                 longitude = NULL,
                                 time = 'file_date'),
              metadata_dims = c('file_date'),
              retrieve = T)
)
  expect_equal(
  names(attr(data, 'Variables')$common),
  c('latitude', 'longitude', 'time', 'tas', NA)
  )
  expect_equal(
  as.vector(attr(data, 'NotFoundFiles')),
  c(NA, "/esarchive/exp/ncep/cfs-v2/weekly_mean/s2s/tas_f24h/tas_20130618.nc")
  )

})

test_that("5. Use parameter 'metadata_dims', all common attributes, no file missing", {
suppressWarnings(
data <- Start(dat = file,
              var = var,
              file_date = sdates5,
              time = indices(1:4),
              latitude = values(list(20, 30)),
              latitude_reorder = Sort(decreasing = TRUE),
              longitude = values(list(-20, -10)),
              longitude_reorder = CircularSort(-180, 180),
              ensemble = indices(1),
              synonims = list(latitude = c('lat', 'latitude'),
                              longitude = c('lon', 'longitude')),
              return_vars = list(latitude = NULL,
                                 longitude = NULL,
                                 time = 'file_date'),
              metadata_dims = c('file_date'),
              retrieve = T)
)
  expect_equal(
  names(attr(data, 'Variables')$common),
  c('latitude', 'longitude', 'time', 'tas', 'tas')
  )
  expect_equal(
  as.vector(attr(data, 'NotFoundFiles')),
  NULL
  )

})
