Newer
Older
context("Start() metadata_dims check")
test_that("1. One data set, one var", {
repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
data <- Start(dat = list(list(name = 'system5_m1', path = repos)),
var = 'tas',
sdate = '20170101',
ensemble = indices(1),
time = indices(1),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
lon = 'dat',
lat = 'dat'),
metadata_dims = 'dat', # it can be omitted since it is automatically specified as 'dat'
retrieve = T
)
expect_equal(
length(attr(data, 'Variables')),
2
)
expect_equal(
names(attr(data, 'Variables')),
c("common", "system5_m1")
)
expect_equal(
names(attr(data, 'Variables')$common),
c('time', 'tas')
)
expect_equal(
names(attr(data, 'Variables')$system5_m1),
c("lon", "lat")
)
expect_equal(
length(attr(data, 'Variables')$common$tas),
12
)
expect_equal(
data[1, 1, 1, 1, 1, , 1],
c(248.5012, 248.7815),
tolerance = 0.0001
)
test_that("2. Two data sets, one var", {
repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
list(name = 'system5_m1', path = repos)),
var = 'tas',
sdate = '20170101',
ensemble = indices(1),
time = indices(1),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
lon = 'dat',
lat = 'dat'),
metadata_dims = 'dat', # it can be omitted since it is automatically specified as 'dat'
retrieve = T
)
expect_equal(
length(attr(data, 'Variables')),
3
)
expect_equal(
names(attr(data, 'Variables')),
c("common", "system4_m1", "system5_m1")
)
expect_equal(
names(attr(data, 'Variables')$common),
'time'
)
expect_equal(
names(attr(data, 'Variables')$system4_m1),
c("lon", "lat", "tas")
)
expect_equal(
names(attr(data, 'Variables')$system5_m1),
c("lon", "lat", "tas")
)
expect_equal(
length(attr(data, 'Variables')$system5_m1$tas),
12
)
expect_equal(
length(attr(data, 'Variables')$system4_m1$tas),
11
)
expect_equal(
data[, 1, 1, 1, 1, 1, 1],
c(247.2570, 248.5012),
tolerance = 0.0001
)
expect_equal(
data[, 1, 1, 1, 1, 1, 2],
c(247.2570, 248.5016),
tolerance = 0.0001
)
test_that("3. One data set, two vars", {
repos <- paste0('/esarchive/exp/ecearth/a1tr/cmorfiles/CMIP/EC-Earth-Consortium/',
'EC-Earth3/historical/r24i1p1f1/Amon/$var$/gr/v20190312/',
'$var$_Amon_EC-Earth3_historical_r24i1p1f1_gr_185001-185012.nc')
var <- c('tas', 'clt')
data <- Start(dat = repos,
var = var,
time = indices(1),
return_vars = list(lat = NULL, lon = NULL),
metadata_dims = 'var',
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
retrieve = TRUE
)
expect_equal(
length(attr(data, 'Variables')),
2
)
expect_equal(
names(attr(data, 'Variables')),
c("common", "dat1")
)
expect_equal(
names(attr(data, 'Variables')$common),
c('lat', 'lon', 'tas', 'clt')
)
expect_equal(
is.null(attr(data, 'Variables')$dat1),
TRUE
)
expect_equal(
length(attr(data, 'Variables')$common$tas),
17
)
expect_equal(
length(attr(data, 'Variables')$common$clt),
16
)
expect_equal(
data[1, , 1, 1, 1],
c(249.42436, 32.45226),
tolerance = 0.0001
)
expect_equal(
data[1, , 1, 2, 1],
c(250.00110, 25.04345),
tolerance = 0.0001
)
})
test_that("4. Two data sets, two vars", {
repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
list(name = 'system5_m1', path = repos)),
var = c('tas', 'sfcWind'),
sdate = '20170101',
ensemble = indices(1),
time = indices(1),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
lon = 'dat',
lat = 'dat'),
metadata_dims = 'dat',
retrieve = T
)
expect_equal(
length(attr(data, 'Variables')),
3
)
expect_equal(
names(attr(data, 'Variables')),
c("common", "system4_m1", "system5_m1")
)
expect_equal(
names(attr(data, 'Variables')$common),
'time'
)
expect_equal(
names(attr(data, 'Variables')$system4_m1),
c("lon", "lat", "tas")
)
expect_equal(
names(attr(data, 'Variables')$system5_m1),
c("lon", "lat", "tas")
)
expect_equal(
length(attr(data, 'Variables')$system5_m1$tas),
12
)
expect_equal(
length(attr(data, 'Variables')$system4_m1$tas),
11
)
expect_equal(
data[1, , 1, 1, 1, 2, 2],
c(247.227219, 6.370782),
tolerance = 0.0001
)
expect_equal(
data[2, , 1, 1, 1, 2, 2],
c(248.781540, 5.794801),
tolerance = 0.0001
)
#-------------------------------------------------------------
data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
list(name = 'system5_m1', path = repos)),
var = c('tas', 'sfcWind'),
sdate = '20170101',
ensemble = indices(1),
time = indices(1),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
lon = 'dat',
lat = 'dat'),
metadata_dims = c('dat', 'var'),
retrieve = T
)
expect_equal(
length(attr(data, 'Variables')),
3
)
expect_equal(
names(attr(data, 'Variables')),
c("common", "system4_m1", "system5_m1")
)
expect_equal(
names(attr(data, 'Variables')$common),
'time'
)
expect_equal(
names(attr(data, 'Variables')$system4_m1),
c("lon", "lat", "tas", "sfcWind")
)
expect_equal(
names(attr(data, 'Variables')$system5_m1),
c("lon", "lat", "tas", "sfcWind")
)
expect_equal(
length(attr(data, 'Variables')$system5_m1$tas),
12
)
expect_equal(
length(attr(data, 'Variables')$system5_m1$sfcWind),
12
)
expect_equal(
length(attr(data, 'Variables')$system4_m1$tas),
11
)
expect_equal(
length(attr(data, 'Variables')$system4_m1$sfcWind),
11
)
expect_equal(
data[1, , 1, 1, 1, 2, 2],
c(247.227219, 6.370782),
tolerance = 0.0001
)
expect_equal(
data[2, , 1, 1, 1, 2, 2],
c(248.781540, 5.794801),
tolerance = 0.0001
)
})
test_that("5. Specify metadata_dims with another file dimension", {
repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
list(name = 'system5_m1', path = repos)),
var = c('tas', 'sfcWind'),
sdate = c('20150101', '20160101', '20170101'),
ensemble = indices(1),
time = indices(1),
lat = indices(1:10),
lon = indices(1:10),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
lon = 'dat',
lat = 'dat'),
metadata_dims = 'sdate',
retrieve = T
)
expect_equal(
length(attr(data, 'Variables')),
3
)
expect_equal(
names(attr(data, 'Variables')),
c("common", "system4_m1", "system5_m1")
)
expect_equal(
names(attr(data, 'Variables')$common),
c('time', 'tas', 'tas', 'tas')
)
expect_equal(
names(attr(data, 'Variables')$system4_m1),
c("lon", "lat")
)
expect_equal(
names(attr(data, 'Variables')$system5_m1),
c("lon", "lat")
)
expect_equal(
length(attr(data, 'Variables')$common$tas),
12
)
})
test_that("6. One data set, two vars from one file", {
mask_path <- '/esarchive/autosubmit/con_files/mask.regions.Ec3.0_O1L46.nc'
data <- Start(repos = mask_path,
var = c('nav_lon', 'nav_lat'),
t = 'first',
z = 'first',
x = 'all',
y = 'all',
return_vars = list(var_names = NULL),
var_var = 'var_names',
retrieve = T)
expect_equal(
length(attr(data, 'Variables')),
2
)
expect_equal(
names(attr(data, 'Variables')),
c("common", "dat1")
)
expect_equal(
names(attr(data, 'Variables')$common),
c("var_names", "nav_lon", "nav_lat")
)
expect_equal(
is.null(attr(data, 'Variables')$dat1),
TRUE
)
expect_equal(
length(attr(data, 'Variables')$common$nav_lat),
8
)
test_that("7. Two data sets, while one is missing", {
repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
# incorrect path. Therefore repos2 doesn't have any valid files
repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f2h/$var$_$sdate$.nc" # correct one is _f6h
var <- 'tas'
suppressWarnings(
data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
list(name = 'system5_m1', path = repos)),
var = var,
sdate = '20170101',
ensemble = indices(1),
time = indices(1),
lat = 'all',
lon = 'all',
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
lon = 'dat',
lat = 'dat'),
metadata_dims = 'dat', # it can be omitted since it is automatically specified as 'dat'
retrieve = T
)
expect_equal(
length(data[is.na(data)]),
829440
)
expect_equal(
attr(data, "Variables")$system4_m1,
list(lon = NULL, lat = NULL)
length(attr(data, "Variables")$system5_m1$lon),
1296
)
expect_equal(
length(attr(data, "Variables")$system5_m1),
3
)
expect_equal(
attr(data, 'Files'),
array(c(NA, "/esarchive/exp/ecmwf/system5_m1/monthly_mean/tas_f6h/tas_20170101.nc"),
dim = c(dat = 2, var = 1, sdate = 1))
)
})
test_that("8. Two data sets, both have files but the first file is missing", {
path_list <- list(
MPI = list(name = 'MPI_ESM',
path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/',
'DCPP/MPI-M/MPI-ESM1-2-HR/',
'dcppA-hindcast/$member$/day/$var$/gn/v20200101/',
'$var$_day_MPI-ESM1-2-HR_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc')),
Had = list(name = 'HadGEM3',
path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/',
'DCPP/MOHC/HadGEM3-GC31-MM/',
'dcppA-hindcast/$member$/day/$var$/gn/v20200101/',
'$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc')))
data <- Start(dataset = path_list,
var = 'tasmin',
member = list(c('r1i1p1f1', 'r2i1p1f2')),
sdate = paste0(2018),
chunk = list(c('20181101-20281231', '20181101-20281230')),
time = indices(1), #'all',
lat = values(list(0, 14)),
lon = values(list(0, 28)),
synonims = list(time = c('fmonth','time'),
lon = c('lon', 'longitude'),
lat = c('lat', 'latitude')),
return_vars = list(lat = 'dataset', lon = 'dataset'),
lat_reorder = Sort(),
num_procs = 1,
retrieve = T)
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
expect_equal(
length(data[is.na(data)]),
5500
)
expect_equal(
length(attr(data, "Variables")$MPI_ESM),
3
)
expect_equal(
length(attr(data, "Variables")$MPI_ESM$lon),
30
)
expect_equal(
names(attr(data, "Variables")$MPI_ESM),
c('lat', 'lon', 'tasmin')
)
expect_equal(
length(attr(data, "Variables")$HadGEM3),
2
)
expect_equal(
length(attr(data, "Variables")$HadGEM3$lon),
34
)
expect_equal(
names(attr(data, "Variables")$HadGEM3),
c('lat', 'lon')
)
expect_equal(
attr(data, 'Files'),
array(c("/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/day/tasmin/gn/v20200101/tasmin_day_MPI-ESM1-2-HR_dcppA-hindcast_s2018-r1i1p1f1_gn_20181101-20281231.nc",
"/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r2i1p1f2/day/tasmin/gn/v20200101/tasmin_day_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r2i1p1f2_gn_20181101-20281230.nc"),