Newer
Older
#*******************************************************************************
# Script to test examples of CST_SaveExp
# Eva Rifà Rovira
# 29/11/2024
#*******************************************************************************
#-------------------------------------------------------------------------------
# Needed packages before a new version is installed
library(CSIndicators)
library(multiApply)
library(easyNCDF)
library(s2dv)
library(ClimProjDiags)
library(CSTools)
library(startR)
source("https://earth.bsc.es/gitlab/external/cstools/-/raw/develop-SaveCube/R/CST_SaveExp.R")
source("https://earth.bsc.es/gitlab/external/cstools/-/raw/develop-SaveCube/R/zzz.R")
################################################################################
# Tests:
#-----------------------------------------------------
# Tests 1: Multidimensional array and Dates, without metadata and coordinates
#-----------------------------------------------------
# (1.1) Minimal use case, without Dates
data <- array(1:5, dim = c(sdate = 5, lon = 4, lat = 4))
SaveExp(data, ftime_dim = NULL, memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, single_file = TRUE)
SaveExp(data, ftime_dim = NULL, memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = NULL, single_file = FALSE) # same result
data <- array(1:5, dim = c(ftime = 5, lon = 4, lat = 4))
SaveExp(data, ftime_dim = 'ftime', memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = NULL, single_file = TRUE)
data <- array(1:5, dim = c(sdate = 5, lon = 4, lat = 4))
SaveExp(data, ftime_dim = NULL, memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = 'sdate', single_file = TRUE)
# (1.4) Only forecast time dimension (no sdate), with Dates
data <- array(1:5, dim = c(ftime = 5, lon = 4, lat = 4))
dates <- c('20000101', '20010102', '20020103', '20030104', '20040105')
dates <- as.Date(dates, format = "%Y%m%d", tz = "UTC")
dim(dates) <- c(ftime = 5)
SaveExp(data, ftime_dim = 'ftime', memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = NULL, Dates = dates, single_file = FALSE)
SaveExp(data, ftime_dim = 'ftime', memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = NULL, Dates = dates, single_file = TRUE)
# For this case we have the same result using: single_file = FALSE /TRUE.
data <- array(1:5, dim = c(sdate = 1, ftime = 5, lon = 4, lat = 4))
dates <- c('20000101', '20010102', '20020103', '20030104', '20040105')
dates <- as.Date(dates, format = "%Y%m%d", tz = "UTC")
dim(dates) <- c(sdate = 1, ftime = 5)
SaveExp(data, ftime_dim = 'ftime', memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = 'sdate', Dates = dates, single_file = FALSE)
SaveExp(data, ftime_dim = 'ftime', memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = 'sdate', Dates = dates, single_file = TRUE)
# (1.6) Test global attributes
SaveExp(data, ftime_dim = 'ftime', memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = 'sdate', Dates = dates, single_file = TRUE,
extra_string = 'test',
global_attrs = list(system = 'tes1', reference = 'test2'))
SaveExp(data, ftime_dim = 'ftime', memb_dim = NULL, dat_dim = NULL,
var_dim = NULL, sdate_dim = 'sdate', Dates = dates, single_file = FALSE,
extra_string = 'test',
global_attrs = list(system = 'tes1', reference = 'test2'))
#-----------------------------------------------------
# Tests 2: Test sample data from Start and from Load
#-----------------------------------------------------
# (2.1) Test SaveExp
exp <- CSTools::lonlat_prec_st
data <- exp$data
Dates = exp$attrs$Dates
coords = exp$coords
varname = exp$attrs$Variable$varName
metadata = exp$attrs$Variable$metadata
SaveExp(data = data, Dates = Dates, coords = coords, varname = varname,
metadata = metadata, ftime_dim = 'ftime', startdates = 1:4,
var_dim = 'var', memb_dim = 'member', dat_dim = 'dataset',
sdate_dim = 'sdate', single_file = FALSE)
SaveExp(data = data, Dates = Dates, coords = coords, varname = varname,
metadata = metadata, ftime_dim = 'ftime', startdates = 1:4,
var_dim = 'var', memb_dim = 'member', dat_dim = 'dataset',
sdate_dim = 'sdate', single_file = TRUE)
# (2.2) lonlat_temp_st$exp in a single file with units 'hours since'
# (2.2.1) We save the data
data <- lonlat_temp_st$exp
CST_SaveExp(data = data, ftime_dim = 'ftime',
var_dim = 'var', dat_dim = 'dataset', sdate_dim = 'sdate',
units_hours_since = TRUE, single_file = TRUE)
sdate <- as.vector(lonlat_temp_st$exp$coords$sdate)
path <- paste0(getwd(),"/$var$_", sdate[1], "_", sdate[length(sdate)], ".nc")
out <- Start(dat = path,
var = 'tas',
member = 'all',
sdate = 'all',
ftime = 'all',
lat = 'all',
lon = 'all',
return_vars = list(lon = 'dat',
lat = 'dat',
ftime = NULL,
sdate = NULL),
retrieve = TRUE)
attributes(out)$Variables$common$ftime
out_cube <- as.s2dv_cube(out)
out_cube <- CST_ChangeDimNames(out_cube,
original_names = c("dat"),
new_names = c("dataset"))
all.equal(data$data, out_cube$data)
identical(data$data, out_cube$data)
# (2.3) lonlat_temp_st$exp in a single file with units of time frequency
# (2.3.1) we save the data
data <- lonlat_temp_st$exp
CST_SaveExp(data = data, ftime_dim = 'ftime',
var_dim = 'var', dat_dim = 'dataset', sdate_dim = 'sdate',
single_file = TRUE, units_hours_since = FALSE)
dates <- lonlat_temp_st$exp$attrs$Dates
sdate <- as.vector(lonlat_temp_st$exp$coords$sdate)
path <- paste0(getwd(),"/$var$_", sdate[1], "_", sdate[length(sdate)], ".nc")
out <- Start(dat = path,
var = 'tas',
lon = 'all',
lat = 'all',
ftime = 'all',
sdate = 'all',
member = 'all',
return_vars = list(
lon = 'dat',
lat = 'dat',
ftime = NULL,
sdate = NULL),
retrieve = TRUE)
attributes(out)$Variables$common$ftime
# [1] "1 months" "2 months" "3 months"
out_cube2 <- as.s2dv_cube(out)
# (2.4) lonlat_temp_st$exp in separated files with units of hours since
# (2.4.1) we save the data
data <- lonlat_temp_st$exp
CST_SaveExp(data = data, ftime_dim = 'ftime',
var_dim = 'var', dat_dim = 'dataset', sdate_dim = 'sdate',
single_file = FALSE)
sdate <- as.vector(lonlat_temp_st$exp$coords$sdate)
path <- paste0(getwd(),"/dat1/$var$/$var$_$sdate$.nc")
out <- Start(dat = path, var = 'tas',
sdate = sdate,
lon = 'all',
lat = 'all',
ftime = 'all',
member = 'all',
return_vars = list(lon = 'dat',
lat = 'dat',
ftime = 'sdate'),
out_cube1 <- as.s2dv_cube(out)
# (2.5) lonlat_prec_st$exp in a single file with units of time frequency
# (2.5.1) we save the data
data <- lonlat_prec_st
CST_SaveExp(data = data, ftime_dim = 'ftime',
var_dim = 'var', dat_dim = 'dataset', sdate_dim = 'sdate',
single_file = TRUE, units_hours_since = FALSE)
sdate <- as.vector(data$coords$sdate)
path <- paste0(getwd(),"/$var$_", sdate[1], "_", sdate[length(sdate)], ".nc")
out <- Start(dat = path,
var = 'prlr',
lon = 'all',
lat = 'all',
ftime = 'all',
sdate = 'all',
member = 'all',
return_vars = list(
lon = 'dat',
lat = 'dat',
ftime = NULL,
sdate = NULL),
retrieve = TRUE)
attributes(out)$Variables$common$ftime
# [1] "1 days" "2 days" "3 days" "4 days" "5 days" "6 days" "7 days"
# [8] "8 days" "9 days" "10 days" "11 days" "12 days" "13 days" "14 days"
# [15] "15 days" "16 days" "17 days" "18 days" "19 days" "20 days" "21 days"
# [22] "22 days" "23 days" "24 days" "25 days" "26 days" "27 days" "28 days"
# [29] "29 days" "30 days" "31 days"
out_cube <- as.s2dv_cube(out)
# (2.6) Test observations: lonlat_temp
# (2.6.1) Save the data
data <- lonlat_temp$obs
CST_SaveExp(data = data, ftime_dim = 'ftime', memb_dim = NULL,
var_dim = NULL, dat_dim = 'dataset', sdate_dim = 'sdate',
single_file = TRUE, units_hours_since = FALSE)
sdate <- c('20001101', '20051101')
path <- paste0(getwd(),"/$var$_", sdate[1], "_", sdate[length(sdate)], ".nc")
out <- Start(dat = path,
var = 'tas', # tas
lon = 'all',
lat = 'all',
ftime = 'all',
member = 1,
sdate = 'all',
return_vars = list(
lon = 'dat',
lat = 'dat',
ftime = NULL,
sdate = NULL),
retrieve = TRUE)
dim(out)
attributes(out)$Variables$common$ftime
# (2.7) Test lonlat_prec
# (2.7.1) Save the data
CST_SaveExp(data = data, ftime_dim = 'ftime', memb_dim = NULL,
var_dim = NULL, dat_dim = 'dataset', sdate_dim = 'sdate',
single_file = TRUE, units_hours_since = FALSE)
sdate <- as.vector(data$coords$sdate)
path <- paste0(getwd(),"/$var$_", sdate[1], "_", sdate[length(sdate)], ".nc")
out <- Start(dat = path,
var = 'prlr', # tas
lon = 'all',
lat = 'all',
ftime = 'all',
sdate = 'all',
member = 'all',
return_vars = list(
lon = 'dat',
lat = 'dat',
ftime = NULL,
sdate = NULL),
retrieve = TRUE)
dim(out)
lonlat_prec$dims
data <- lonlat_temp$exp
data <- CST_Subset(data, along = 'ftime', indices = 1, drop = 'selected')
CST_SaveExp(data = data, ftime_dim = NULL,
var_dim = NULL, dat_dim = 'dataset', sdate_dim = 'sdate',
single_file = FALSE, units_hours_since = FALSE)
#-----------------------------------------------------
#-----------------------------------------------------
# (3.1) Two variables and two datasets in separated files
# (3.1.1) We load the data from Start
repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
data3 <- Start(dat = list(list(name = 'system4_m1', path = repos2),
list(name = 'system5_m1', path = repos)),
var = c('tas', 'sfcWind'),
sdate = c('20160101', '20170101'),
ensemble = indices(1),
time = indices(1:2),
lat = indices(1:10),
lon = indices(1:10),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
longitude = 'dat',
latitude = 'dat'),
metadata_dims = c('dat', 'var'),
retrieve = T)
cube3 <- as.s2dv_cube(data3)
CST_SaveExp(data = cube3, ftime_dim = 'time', var_dim = 'var',
memb_dim = 'ensemble', dat_dim = 'dat')
repos <- paste0(getwd(), "/system4_m1/$var$/$var$_$sdate$.nc")
repos2 <- paste0(getwd(), "/system5_m1/$var$/$var$_$sdate$.nc")
data3out <- Start(dat = list(list(name = 'system4_m1', path = repos2),
list(name = 'system5_m1', path = repos)),
var = c('tas', 'sfcWind'),
sdate = c('20160101', '20170101'),
ensemble = indices(1),
time = indices(1:2),
lat = indices(1:10),
lon = indices(1:10),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(time = 'sdate',
longitude = 'dat',
latitude = 'dat'),
metadata_dims = c('dat', 'var'),
retrieve = T)
summary(data3out)
summary(data3)
dim(data3)
dim(data3out)
# (3.2) Two variables and two datasets in the same file
CST_SaveExp(data = cube3, ftime_dim = 'time', var_dim = 'var',
memb_dim = 'ensemble', dat_dim = 'dat',
single_file = TRUE)
# TODO: Read the output with Start
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
repos_exp <- paste0('/esarchive/exp/ecearth/a1tr/cmorfiles/CMIP/EC-Earth-Consortium/',
'EC-Earth3/historical/r24i1p1f1/Amon/$var$/gr/v20190312/',
'$var$_Amon_EC-Earth3_historical_r24i1p1f1_gr_$sdate$01-$sdate$12.nc')
exp <- Start(dat = repos_exp,
var = 'tas',
sdate = as.character(c(2005:2008)),
time = indices(1:3),
lat = 1:10,
lat_reorder = Sort(),
lon = 1:10,
lon_reorder = CircularSort(0, 360),
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(lon = NULL,
lat = NULL,
time = 'sdate'),
retrieve = FALSE)
dates <- attr(exp, 'Variables')$common$time
repos_obs <- '/esarchive/recon/ecmwf/erainterim/monthly_mean/$var$_f6h/$var$_$date$.nc'
obs <- Start(dat = repos_obs,
var = 'tas',
date = unique(format(dates, '%Y%m')),
time = values(dates), #dim: [sdate = 4, time = 3]
lat = 1:10,
lat_reorder = Sort(),
lon = 1:10,
lon_reorder = CircularSort(0, 360),
time_across = 'date',
merge_across_dims = TRUE,
split_multiselected_dims = TRUE,
synonims = list(lat = c('lat', 'latitude'),
lon = c('lon', 'longitude')),
return_vars = list(lon = NULL,
lat = NULL,
time = 'date'),
retrieve = TRUE)
obscube <- as.s2dv_cube(obs)
CST_SaveExp(data = obscube, ftime_dim = 'time', var_dim = 'var',
memb_dim = NULL, dat_dim = 'dat',
single_file = TRUE, extra_string = 'obs_tas')
CST_SaveExp(data = obscube, ftime_dim = 'time', var_dim = 'var',
memb_dim = NULL, dat_dim = 'dat',
single_file = FALSE, extra_string = 'obs_tas')
# example: /esarchive/exp/ncep/cfs-v2/weekly_mean/s2s/tas_f24h/tas_20231128.nc
library(CSIndicators)
exp <- CSTools::lonlat_prec_st
exp$attrs$Dates <- Reorder(exp$attrs$Dates, c(2,1))
res <- CST_PeriodAccumulation(data = exp, time_dim = 'ftime',
start = list(10, 03), end = list(20, 03))
# > dim(res$attrs$Dates)
# sdate
# 3
# (4.1) All data in a single file
CST_SaveExp(data = res, ftime_dim = NULL, var_dim = 'var',
memb_dim = 'member', dat_dim = 'dataset',
startdates = res$attrs$Dates, single_file = TRUE)
# (4.1.1) Same with SaveExp
SaveExp(data = res$data, coords = res$coords,
Dates = NULL, time_bounds = res$attrs$time_bounds,
ftime_dim = NULL, var_dim = 'var',
varname = res$attrs$Variable$varName,
metadata = res$attrs$Variable$metadata,
memb_dim = 'member', dat_dim = 'dataset',
startdates = res$attrs$Dates, single_file = TRUE)
# (4.2) All data in separated files
CST_SaveExp(data = res, ftime_dim = NULL, var_dim = 'var',
memb_dim = 'member', dat_dim = 'dataset',
startdates = res$attrs$Dates, single_file = FALSE)
# (4.2.1) Same with SaveExp
SaveExp(data = res$data, coords = res$coords,
Dates = res$attrs$Dates, time_bounds = res$attrs$time_bounds,
ftime_dim = NULL, var_dim = 'var',
varname = res$attrs$Variable$varName,
metadata = res$attrs$Variable$metadata,
memb_dim = 'member', dat_dim = 'dataset',
startdates = res$attrs$Dates, single_file = FALSE)
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
# (4.3)
CST_SaveExp(data = res, ftime_dim = NULL, var_dim = 'var',
memb_dim = 'member', dat_dim = 'dataset',
startdates = 1:4, single_file = FALSE)
# (4.4) We change the time dimensions to ftime and sdate_dim = NULL
dim(res$attrs$time_bounds[[1]]) <- c(time = 3)
dim(res$attrs$time_bounds[[2]]) <- c(time = 3)
dim(res$attrs$Dates) <- c(time = 3)
dim(res$data) <- c(dataset = 1, var = 1, member = 6, time = 3, lat = 4, lon = 4)
# (4.4.1) All data in a single file
CST_SaveExp(data = res, ftime_dim = 'time', var_dim = 'var',
memb_dim = 'member', dat_dim = 'dataset', sdate_dim = NULL,
startdates = res$attrs$Dates, single_file = TRUE)
# (4.4.2) All data in separated files
CST_SaveExp(data = res, ftime_dim = 'time', var_dim = 'var',
memb_dim = 'member', dat_dim = 'dataset', sdate_dim = NULL,
startdates = res$attrs$Dates, single_file = FALSE)
# (4.5) Forecast time units
CST_SaveExp(data = res, ftime_dim = 'time', var_dim = 'var',
memb_dim = 'member', dat_dim = 'dataset', sdate_dim = NULL,
startdates = res$attrs$Dates, single_file = TRUE,
units_hours_since = FALSE)
#-----------------------------------------------------
#-----------------------------------------------------
data <- lonlat_temp$exp
# data <- lonlat_temp$obs
# data <- lonlat_prec
CST_SaveExp(data = data, ftime_dim = 'ftime',
var_dim = NULL, dat_dim = 'dataset', sdate_dim = 'sdate',
single_file = FALSE, units_hours_since = FALSE)
# Now we read the output with Load:
# startDates <- c('20001101', '20011101', '20021101',
# '20031101', '20041101', '20051101')
# infile <- list(path = paste0(getwd(),
# '/system5c3s/$VAR_NAME$/$VAR_NAME$_$START_DATE$.nc'))
# out_lonlat_temp <- CST_Load(var = 'tas', exp = list(infile), obs = NULL,
# sdates = startDates,
# nmember = 15,
# leadtimemax = 3,
# latmin = 27, latmax = 48,
# lonmin = -12, lonmax = 40,
# output = "lonlat")
# Error
################################################################################