diff --git a/.Rbuildignore b/.Rbuildignore index 15222c0c49a1e16886a535ce490d47940170e23c..b4bd7473ad1a0c485b15e042bd53ddcae7e74b89 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -6,9 +6,4 @@ .*^(?!data)\.RData$ .*\.gitlab-ci.yml$ ^tests$ -#^inst/doc$ -^inst/doc/UseCase2_PrecipitationDownscaling_RainFARM_RF100\.R$ -^inst/doc/UseCase1_WindEvent_March2018\.R$ -^inst/doc/UseCase2_PrecipitationDownscaling_RainFARM_RF4\.R$ -^inst/doc/UseCase3_data_preparation_SCHEME_model\.R$ -^inst/doc/launch_UseCase2_PrecipitationDownscaling_RF4\.sh$ +^inst/doc$ diff --git a/DESCRIPTION b/DESCRIPTION index d24003fdbedcc573c8250c1599f85deebba96cf2..53fdebdea9459beaafec17f7034dce74e931ff77 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: CSTools Title: Assessing Skill of Climate Forecasts on Seasonal-to-Decadal Timescales -Version: 4.1.1 +Version: 5.0.0 Authors@R: c( person("Nuria", "Perez-Zanon", , "nuria.perez@bsc.es", role = "aut", comment = c(ORCID = "0000-0001-8568-3071")), person("Louis-Philippe", "Caron", , "louis-philippe.caron@bsc.es", role = "aut", comment = c(ORCID = "0000-0001-5221-0147")), @@ -88,7 +88,7 @@ Suggests: rmarkdown, startR VignetteBuilder: knitr -License: Apache License 2.0 +License: GPL-3 Encoding: UTF-8 LazyData: true RoxygenNote: 7.2.0 diff --git a/NEWS.md b/NEWS.md index 72362e9bbbc6092b81aca5a1a95d0ce1428ffffd..2c6344004db4899765ee5ba9368a3d1dc80a67cc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,139 +1,147 @@ -### CSTools 4.1.1 -**Submission date to CRAN: 10-11-2022** -- Fixes: - + CST_Analogs corrected input of ClimProjDiags::Subset() - + PlotCombinedMap corrected use of 'cex_bar_titles' parameter - + CST_Anomaly added 'memb_dim', 'dat_dim' and 'ftime_dim' and improved use for 'dim_anom' parameters - -### CSTools 4.1.0 -**Submission date to CRAN: 25-10-2022** -- New features: - + Dependency on package 's2dverification' is changed to 's2dv' - + CST_BiasCorrection new parameters 'memb_dim', 'sdate_dim', 'ncores' - + CST_Calibration is able to calibrate forecast with new parameter 'exp_cor' - + CST_QuantileMapping uses cross-validation and provides option to remove NAs; new parameters 'memb_dim', 'sdate_dim', 'window_dim' and 'na.rm'; 'sample_dim' and 'sample_length' are removed - + s2dv_cube() new parameter 'time_dim' - -- Fixes: - + as.s2dv_cube() detects latitude and longitude structure in startR_array object - + Data correction: 'lonlat_data' is renamed to 'lonlat_temp'; 'lonlat_prec' is corrected by one-day shift - + Typo and parameter correction in vignette 'MostLikelyTercile_vignette' - + Figure and result correction in vignette 'RainFARM_vignette' - + PlotMostLikelyQuantileMap() works with s2dv::PlotLayout - -### CSTools 4.0.1 -**Submission date to CRAN: 05-10-2021** - -- New features: - + Dynamical Bias Correction method: `CST_ProxiesAttractors` and `CST_DynBiasCorrection` - (optionally `Predictability`) - + CST_BiasCorrection and BiasCorrection allows to calibrate a forecast given the calibration in the hindcast by using parameter 'exp_cor'. - + Use cases - + CST_SaveExp includes parameter extra_string - + PlotCombinedMap includes parameter cex_bar_titles - -- Fixes: - + Calibration retains correlation absolute value - + Calibration fixed when cal.methodi == rpc-based, apply_to == sign, - eval.method == 'leave-one-out' and the correlation is not significant - + PlotMostLikelyQuantileMap reoder latitudes of an array provided in 'dots' parameter. - -### CSTools 4.0.0 -**Submission date to CRAN: 23-02-2021** - -- New features: - + ADAMONT downscaling method: requires CST_AdamontAnalogs and CST_AdamontQQCor functions - + Analogs method using Predictors: requires training_analogs and CST_AnalogsPredictors - + PlotPDFsOLE includes parameters to modify legend style - + CST_RFSlope handless missing values in the temporal dimension and new 'ncores' parameter allows parallel computation - + CST_RFWeights accepts s2dv_cube objects as input and new 'ncores' paramenter allows parallel computation - + RFWeights is exposed to users - + CST_RainFARM accepts multi-dimensional slopes and weights and handless missing values in sample dimensions. - + QuantileMapping is exposed to users - + CST_MultiMetric includes 'rpss' metric and it is addapted to s2dv. - + PlotMostLikelyQuantileMap vignette - + PlotTriangles4Categories includes two parameters to adjust axis and margins - + CategoricalEnsCombination is exposed to users - + CST_SplitDims includes parameter 'insert_ftime' - + Analogs vignette - + Data Storage and retrieval vignette - -- Fixes: - + PlotForecastPDF correctly displays terciles labels - + CST_SaveExp correctly save time units - + CST_SplitDims returns ordered output following ascending order provided in indices when it is numeric - + qmap library moved from Imports to Depends - + CST_QuantileMapping correctly handles exp_cor - + Figures resize option from vignettes has been removed - + Fix Analogs to work with three diferent criteria - + Vignette PlotForecastPDF updated plots - + Decrease package size compresing vignettes figures and removing areave_data sample - -### CSTools 3.1.0 -**Submission date to CRAN: 02-07-2020** - -- New features: - + EnsClustering vignette - + EnsClustering has a new parameter 'time_dim' - + CST_BiasCorrection has na.rm paramter - + CST_Anomaly allows to smooth the climatology with filter.span parameter - + PlotTriangles4Categories new plotting function to convert any 3-d numerical array to a grid of coloured triangles. - + CST_WeatherRegimes/WeatherRegimes and CST_RegimeAssign/RegimeAssign - + PlotPDFsOLE plots two probability density gaussian functions and the optimal linear estimation - + CST_RFTemp/RF_Temp functions available for downscaling temperature - + Weather Regimes vignette - -- Fixes - + CST_Anomaly handles exp, obs or both - + PlotForecastPDF vignette displays figures correctly - + Calibration function is exposed to users - + MultiMetric vignette fixed typo text description - + RainFARM checks 'slope' is not a vector - + DESCRIPTION specifies the minimum multiApply version required - + EnsClustering has a fixed 'closest_member' output - + PlotCombinedMap handles masks correctly - + CST_SaveExp uses multiApply and save time dimension correctly - -### CSTools 3.0.0 -**Submission date to CRAN: 10-02-2020** - -- New features: - + CST_MergeDims and MergeDims - + Version working with R 3.4.2 - + PlotForecastPDF handles independent terciles, extremes and observations for each panel -- Fixes - + CST_Calibration handles missing values - + BEI functions handle missing values - - -### CSTools 2.0.0 -**Submission date to CRAN: 25-11-2019** - -- New features: - + CST_Analogs Analogs downscaling method, - + CST_MultiEOFS for multiple variables, - + Ensemble Clustering, - + Categorical Ensemble Combination, - + new Calibration methods included in CST_Calibration, - + Best Estimated Index method, - + CST_QuantileMapping, - + CST_SplitDim to split dimension, if it is a temporal dimension, it can be split by days, months and years or other inidices, - + creation and transformation to class 's2dv_cube', - + CST_SaveExp function for saving experiments to be loadable with CST_Load, +# CSTools 5.0.0 (Release date: 28-03-2023) +**Fixes** +- Correct vignettes: Analogs, MultiModelSkill and MultivarRMSE +- Add 'ncores' to s2dv function calls in CST_Anomaly +- Reduce computing time of examples and tests and improve documentation + +**New features** +- Add dat_dim parameter in CST_BiasCorrection and CST_Calibration +- New plotting function for case studies temporal visualisation: PlotWeeklyClim +- Deprecate indices in dim_anom parameter of CST_Anomaly +- Allow memb_dim to be NULL in QuantileMapping +- Uncomment tests in CST_MultivarRMSE due to correction of RMS in s2dv next release (released s2dv 1.4.0 21/03) +- New s2dv_cube object development for all the functions, unit tests, examples and vignettes +- New function CST_Subset similar to Subset with 's2dv_cubes' +- Improved CST_SaveExp function with new features +- New color set in PlotForecastPDF Vitigeoss colors + +**Other** +- Added contribution from ArticXchange project due to PlotWeeklyClim +- Update NEWS.md with the correct format +- Change Licence + +# CSTools 4.1.1 (Release date: 10-11-2022) +**Fixes** +- CST_Analogs corrected input of ClimProjDiags::Subset() +- PlotCombinedMap corrected use of 'cex_bar_titles' parameter +- CST_Anomaly added 'memb_dim', 'dat_dim' and 'ftime_dim' and improved use for 'dim_anom' parameters + +# CSTools 4.1.0 (Release date: 25-10-2022) +**New features** +- Dependency on package 's2dverification' is changed to 's2dv' +- CST_BiasCorrection new parameters 'memb_dim', 'sdate_dim', 'ncores' +- CST_Calibration is able to calibrate forecast with new parameter 'exp_cor' +- CST_QuantileMapping uses cross-validation and provides option to remove NAs; new parameters 'memb_dim', 'sdate_dim', 'window_dim' and 'na.rm'; 'sample_dim' and 'sample_length' are removed +- s2dv_cube() new parameter 'time_dim' + +**Fixes** +- as.s2dv_cube() detects latitude and longitude structure in startR_array object +- Data correction: 'lonlat_data' is renamed to 'lonlat_temp'; 'lonlat_prec' is corrected by one-day shift +- Typo and parameter correction in vignette 'MostLikelyTercile_vignette' +- Figure and result correction in vignette 'RainFARM_vignette' +- PlotMostLikelyQuantileMap() works with s2dv::PlotLayout + +# CSTools 4.0.1 (Release date: 05-10-2021) +**New features** +- Dynamical Bias Correction method: `CST_ProxiesAttractors` and `CST_DynBiasCorrection` (optionally `Predictability`) +- CST_BiasCorrection and BiasCorrection allows to calibrate a forecast given the calibration in the hindcast by using parameter 'exp_cor'. +- Use cases +- CST_SaveExp includes parameter extra_string +- PlotCombinedMap includes parameter cex_bar_titles + +**Fixes** +- Calibration retains correlation absolute value +- Calibration fixed when cal.methodi == rpc-based, apply_to == sign, eval.method == 'leave-one-out' and the correlation is not significant +- PlotMostLikelyQuantileMap reoder latitudes of an array provided in 'dots' parameter. + +# CSTools 4.0.0 (Release date: 23-02-2021) +**New features** +- ADAMONT downscaling method: requires CST_AdamontAnalogs and CST_AdamontQQCor functions +- Analogs method using Predictors: requires training_analogs and CST_AnalogsPredictors +- PlotPDFsOLE includes parameters to modify legend style +- CST_RFSlope handless missing values in the temporal dimension and new 'ncores' parameter allows parallel computation +- CST_RFWeights accepts s2dv_cube objects as input and new 'ncores' paramenter allows parallel computation +- RFWeights is exposed to users +- CST_RainFARM accepts multi-dimensional slopes and weights and handless missing values in sample dimensions. +- QuantileMapping is exposed to users +- CST_MultiMetric includes 'rpss' metric and it is addapted to s2dv. +- PlotMostLikelyQuantileMap vignette +- PlotTriangles4Categories includes two parameters to adjust axis and margins +- CategoricalEnsCombination is exposed to users +- CST_SplitDims includes parameter 'insert_ftime' +- Analogs vignette +- Data Storage and retrieval vignette + +**Fixes** +- PlotForecastPDF correctly displays terciles labels +- CST_SaveExp correctly save time units +- CST_SplitDims returns ordered output following ascending order provided in indices when it is numeric +- qmap library moved from Imports to Depends +- CST_QuantileMapping correctly handles exp_cor +- Figures resize option from vignettes has been removed +- Fix Analogs to work with three diferent criteria +- Vignette PlotForecastPDF updated plots +- Decrease package size compresing vignettes figures and removing areave_data sample + +# CSTools 3.1.0 (Release date: 02-07-2020) +**New features** +- EnsClustering vignette +- EnsClustering has a new parameter 'time_dim' +- CST_BiasCorrection has na.rm paramter +- CST_Anomaly allows to smooth the climatology with filter.span parameter +- PlotTriangles4Categories new plotting function to convert any 3-d numerical array to a grid of coloured triangles. +- CST_WeatherRegimes/WeatherRegimes and CST_RegimeAssign/RegimeAssign +- PlotPDFsOLE plots two probability density gaussian functions and the optimal linear estimation +- CST_RFTemp/RF_Temp functions available for downscaling temperature +- Weather Regimes vignette + +**Fixes** +- CST_Anomaly handles exp, obs or both +- PlotForecastPDF vignette displays figures correctly +- Calibration function is exposed to users +- MultiMetric vignette fixed typo text description +- RainFARM checks 'slope' is not a vector +- DESCRIPTION specifies the minimum multiApply version required +- EnsClustering has a fixed 'closest_member' output +- PlotCombinedMap handles masks correctly +- CST_SaveExp uses multiApply and save time dimension correctly + +# CSTools 3.0.0 (Release date: 10-02-2020) +**New features** +- CST_MergeDims and MergeDims +- Version working with R 3.4.2 +- PlotForecastPDF handles independent terciles, extremes and observations for each panel + +**Fixes** +- CST_Calibration handles missing values +- BEI functions handle missing values + +# CSTools 2.0.0 (Release date: 25-11-2019) +**New features** +- CST_Analogs Analogs downscaling method, +- CST_MultiEOFS for multiple variables, +- Ensemble Clustering, +- Categorical Ensemble Combination, +- new Calibration methods included in CST_Calibration, +- Best Estimated Index method, +- CST_QuantileMapping, +- CST_SplitDim to split dimension, if it is a temporal dimension, it can be split by days, months and years or other inidices, +- creation and transformation to class 's2dv_cube', +- CST_SaveExp function for saving experiments to be loadable with CST_Load, - Parallelization of RainFARM downscaling - Adding unit tests using testthat for BEI and RainFarm functions - New vignette Best Estimate Index -- Minor fix in CST_BiasCorrection when checking parameter 'obs' - Addapting CST_Load to use 'as.s2dv_cube' function -- Minor fix in data lonlat_prec to be of class 's2dv_cube' -- Minor fix in RainFARM vignette - Adding reference to S2S4E H2020 project into the DESCRIPTION file - Adding NEWS.md file +**Fixes** +- Minor fix in CST_BiasCorrection when checking parameter 'obs' +- Minor fix in data lonlat_prec to be of class 's2dv_cube' +- Minor fix in RainFARM vignette -### CSTools 1.0.1 -**Release date on CRAN: 19-06-2019** - +### CSTools 1.0.1 (Release date: 19-06-2019) +**Fixes and new features** - Correcting test of PlotForecastPDF for compatibility with ggplot2 release - New function PlotCombinedMap - Adding reference to MEDSCOPE ERA4CS Project into the DESCRIPTION file @@ -141,11 +149,7 @@ - Minor fix in PlotMostLikelyQuantileMap for bar_titles - MultiModelSkill vignette updated to use PlotCombinedMap - - -### CSTools 1.0.0 -**Release date on CRAN: 24-04-2019** - +### CSTools 1.0.0 (Release date: 24-04-2019) - Features included: Load, Anomaly, MultiMetric, MultivarRMSE, Calibration, BiasCorrection, RainFARM Downscaling, PlotForecastPDF, PlotMostLikelyQuantileMap - Three sample data: lonlat_data, lonlat_prec, areave_data - Unit tests using testthat: BiasCorrection, Calibration, MultiMetric, PlotForecast diff --git a/R/CST_SaveExp.R b/R/CST_SaveExp.R index 91b0a78c16736b363dbf76b1301e02877ee0212a..1c419be56453ef5e9515cd4a0b7130a4b0826ec4 100644 --- a/R/CST_SaveExp.R +++ b/R/CST_SaveExp.R @@ -159,7 +159,7 @@ CST_SaveExp <- function(data, destination = "./", sdate_dim = 'sdate', #'@param data A multi-dimensional array with named dimensions. #'@param destination A character string indicating the path where to store the #' NetCDF files. -#'@param Dates An named array of dates with the corresponding sdate and forecast +#'@param Dates A named array of dates with the corresponding sdate and forecast #' time dimension. #'@param coords A named list with elements of the coordinates corresponding to #' the dimensions of the data parameter. The names and length of each element @@ -198,14 +198,25 @@ CST_SaveExp <- function(data, destination = "./", sdate_dim = 'sdate', #' for instance, to identify member or realization. It would be added to the #' file name between underscore characters. #' -#'@return If single_file is TRUE only one file is created. If single_file is -#'FALSE multiple files are created. When multiple files are created, each file -#'contains the data subset for each start date, variable and dataset. Files -#'with different variables and Datasets are stored in separated directories. -#'The path will be created with the name of the variable and each start date. -#'NetCDF file for each starting date are saved into the -#' folder tree: \cr -#' destination/Dataset/variable/. +#'@return Multiple or single NetCDF files containing the data array.\cr +#'\item{\code{single_file = TRUE}}{ +#' All data is saved in a single file located in the specified destination +#' path with the following name: +#' ___.nc. Multiple +#' variables are saved separately in the same file. The forecast time units +#' is extracted from the frequency of the time steps (hours, days, months). +#' The first value of forecast time is 1. If no frequency is found, the units +#' will be 'hours since' each start date and the time steps are assumed to be +#' equally spaced. +#'} +#'\item{\code{single_file = FALSE}}{ +#' The data array is subset and stored into multiple files. Each file +#' contains the data subset for each start date, variable and dataset. Files +#' with different variables and Datasets are stored in separated directories +#' within the following directory tree: destination/Dataset/variable/. +#' The name of each file will be: +#' __.nc. +#'} #' #'@examples #'\dontrun{ @@ -223,6 +234,7 @@ CST_SaveExp <- function(data, destination = "./", sdate_dim = 'sdate', #' metadata = metadata, single_file = TRUE, ftime_dim = 'ftime', #' var_dim = NULL) #'} +#' #'@import ncdf4 #'@importFrom s2dv Reorder #'@import multiApply diff --git a/README.md b/README.md index 65a78d80ce1b71d0628637019cf48437adc11066..abfc47146ebcf58691715da988d19e53dc614a20 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,22 @@ -Welcome to the CSTools GitLab website -====================================== +CSTools +======= -The Climate Services Tools, CSTools, is an easy-to-use R package designed and built to assess and improve the quality of climate forecasts for seasonal to multi–annual scales. The package contains process-based state-of-the-art methods for forecast calibration, bias correction, statistical and stochastic downscaling, optimal forecast combination and multivariate verification, as well as basic and advanced tools to obtain tailored products. +The Climate Services Tools, CSTools, is an easy-to-use R package designed and built to assess and improve the quality of climate forecasts for seasonal to multi–annual scales. The package contains process-based state-of-the-art methods for forecast calibration, bias correction, statistical and stochastic downscaling, optimal forecast combination and multivariate verification, as well as basic and advanced tools to obtain tailored products. This package was developed in the context of the ERA4CS project MEDSCOPE and the H2020 S2S4E project and includes contributions from ArticXchange project founded by EU-PolarNet 2. This GitLab project allows you to monitor its progress and to interact with other developers via the Issues section. -A scientific publication including use cases was published in the Geoscientific Model Development Journal, and it can be cited as follows: -> Pérez-Zanón, N., Caron, L.-P., Terzago, S., Van Schaeybroeck, B., Lledó, L., Manubens, N., Roulin, E., Alvarez-Castro, M. C., Batté, L., Bretonnière, P.-A., Corti, S., Delgado-Torres, C., Domínguez, M., Fabiano, F., Giuntoli, I., von Hardenberg, J., Sánchez-García, E., Torralba, V., and Verfaillie, D.: Climate Services Toolbox (CSTools) v4.0: from climate forecasts to climate forecast information, Geosci. Model Dev., 15, 6115–6142, https://doi.org/10.5194/gmd-15-6115-2022, 2022. +A scientific publication including use cases was published in the Geoscientific Model Development Journal, and it can be cited as follows: +> Pérez-Zanón, N., Caron, L.-P., Terzago, S., Van Schaeybroeck, B., Lledó, L., Manubens, N., Roulin, E., Alvarez-Castro, M. C., Batté, L., Bretonnière, P.-A., Corti, S., Delgado-Torres, C., Domínguez, M., Fabiano, F., Giuntoli, I., von Hardenberg, J., Sánchez-García, E., Torralba, V., and Verfaillie, D.: Climate Services Toolbox (CSTools) v4.0: from climate forecasts to climate forecast information, Geosci. Model Dev., 15, 6115–6142, https://doi.org/10.5194/gmd-15-6115-2022, 2022. On-line resources ----------------- A part from this GitLab project, that allows you to monitor CSTools progress, to interact with other developers via the Issues section and to contribute, you can find: -- The CRAN repository which includes the user manual and vignettes - -- Video tutorials - -- Other resources are under-development such [training material](https://earth.bsc.es/gitlab/external/cstools/-/tree/MEDCOF2022/inst/doc/MEDCOF2022) and a [full reproducible use case for forecast calibration](https://earth.bsc.es/gitlab/external/cstools/-/tree/develop-CalibrationVignette/FOCUS_7_2) +- The CRAN repository [https://CRAN.R-project.org/package=CSTools](https://CRAN.R-project.org/package=CSTools) which includes the user manual and vignettes. +- Video tutorials [https://www.medscope-project.eu/products/tool-box/cstools-video-tutorials/](https://www.medscope-project.eu/products/tool-box/cstools-video-tutorials/). +- Other resources are under-development such [training material](https://earth.bsc.es/gitlab/external/cstools/-/tree/MEDCOF2022/inst/doc/MEDCOF2022) and a [full reproducible use case for forecast calibration](https://earth.bsc.es/gitlab/external/cstools/-/tree/develop-CalibrationVignette/FOCUS_7_2). Installation ------------ @@ -26,22 +24,66 @@ Installation CSTools has a system dependency, the CDO libraries, for interpolation of grid data and retrieval of metadata. Make sure you have these libraries installed in the system or download and install from -. +[https://code.zmaw.de/projects/cdo](https://code.zmaw.de/projects/cdo). You can then install the public released version of CSTools from CRAN: + ```r install.packages("CSTools") ``` + Or the development version from the GitLab repository: + ```r # install.packages("devtools") devtools::install_git("https://earth.bsc.es/gitlab/external/cstools.git") ``` -How to contribute ------------------ +Overview +-------- + +The CSTools package functions can be distributed in the following methods: + +- **Data retrieval and formatting:** CST_Load, CST_Anomaly, CST_MergeDims, CST_SplitDims, CST_Subset, as.s2dv_cube, s2dv_cube, CST_SaveExp. +- **Classification:** CST_MultiEOF, CST_WeatherRegimes, CST_RegimsAssign, CST_CategoricalEnsCombination, CST_EnsClustering. +- **Downscaling:** CST_Analogs, CST_RainFARM, CST_RFTemp, CST_AdamontAnalog, CST_AnalogsPredictors. +- **Correction:** CST_BEI_Weighting, CST_BiasCorrection, CST_Calibration, CST_QuantileMapping, CST_DynBiasCorrection. +- **Assessment:** CST_MultiMetric, CST_MultivarRMSE +- **Visualization:** PlotCombinedMap, PlotForecastPDF, PlotMostLikelyQuantileMap, PlotPDFsOLE, PlotTriangles4Categories, PlotWeeklyClim. -Before adding a development, we suggest to contact the package mantainer. Details on the procedure and development guidelines can be found in [this issue](https://earth.bsc.es/gitlab/external/cstools/-/issues/3) +[CSTools](https://cran.r-project.org/web/packages/CSTools/index.html) package is designed to be compatible with other R packages such as [s2dv](https://cran.r-project.org/web/packages/s2dv/index.html), [startR](https://cran.r-project.org/web/packages/startR/index.html), [CSIndicators](https://cran.r-project.org/web/packages/CSIndicators/index.html), [CSDownscale](https://earth.bsc.es/gitlab/es/csdownscale). Functions with the prefix **CST_** deal with a common object called `s2dv_cube` as inputs. Also, this object can be created from Load (s2dv) and from Start (startR) directly. Multiple functions from different packages can operate on this common data structure to easily define a complete post-processing workflow. + +The class `s2dv_cube` is mainly a list of named elements to keep data and metadata in a single object. Basic structure of the object class `s2dv_cube`: + +```r +$data: A multidimensional array +$dims: Dimensions vector +$coords: Named list of coordinates vector + $sdate + $time + $lon + [...] +$attrs: Named list containing the metadata + $Variable + $Datasets + $source_files + $when + $Dates + $load_parameters +``` + +More information about the `s2dv_cube` object class can be found here: [Description of the s2dv_cube object structure document](https://docs.google.com/document/d/1ko37JFl_h6mOjDKM5QSQGikfLBKZq1naL11RkJIwtMM/edit?usp=sharing). + +The current `s2dv_cube` object (CSTools 5.0.0) differs from the original object (used in CSTools v < 5.0.0). If you have **questions** on this change you can follow some of the points below: + +- [New s2dv_cube object discussion](https://earth.bsc.es/gitlab/external/cstools/-/issues/94) +- [How to deal with the compatibility break](https://earth.bsc.es/gitlab/external/cstools/-/issues/112) +- [Testing issue and specifications](https://earth.bsc.es/gitlab/external/cstools/-/issues/110) + +Contribute +---------- + +Before adding a development, we suggest to contact the package mantainer. Details on the procedure and development guidelines can be found in [this issue](https://earth.bsc.es/gitlab/external/cstools/-/issues/3). If you plan on contributing, you should rather clone the project on your workstation and modify it using the basic Git commands (clone, branch, add, commit, push, merge, ...). @@ -49,6 +91,5 @@ The code of each function should live in a separate file with the .R extension u For an introductory video on Git, you can have a look at https://vimeo.com/41027679. -You can also find all the necessary documentation on git here: https://git-scm.com/book/en/v2 +You can also find all the necessary documentation on git here: https://git-scm.com/book/en/v2 A lot of it may be a bit complicated for beginners (and not necessary for us), but the "Getting started" and "Git basics" sections are a good resources. - diff --git a/man/SaveExp.Rd b/man/SaveExp.Rd index 6ddb4afb2cc729f36a15012f0c053e9709ecb92c..d0418131c288acd2626fb90b3b76c58cbca8a0ec 100644 --- a/man/SaveExp.Rd +++ b/man/SaveExp.Rd @@ -28,7 +28,7 @@ SaveExp( \item{destination}{A character string indicating the path where to store the NetCDF files.} -\item{Dates}{An named array of dates with the corresponding sdate and forecast +\item{Dates}{A named array of dates with the corresponding sdate and forecast time dimension.} \item{coords}{A named list with elements of the coordinates corresponding to @@ -80,14 +80,25 @@ for instance, to identify member or realization. It would be added to the file name between underscore characters.} } \value{ -If single_file is TRUE only one file is created. If single_file is -FALSE multiple files are created. When multiple files are created, each file -contains the data subset for each start date, variable and dataset. Files -with different variables and Datasets are stored in separated directories. -The path will be created with the name of the variable and each start date. -NetCDF file for each starting date are saved into the - folder tree: \cr - destination/Dataset/variable/. +Multiple or single NetCDF files containing the data array.\cr +\item{\code{single_file = TRUE}}{ + All data is saved in a single file located in the specified destination + path with the following name: + ___.nc. Multiple + variables are saved separately in the same file. The forecast time units + is extracted from the frequency of the time steps (hours, days, months). + The first value of forecast time is 1. If no frequency is found, the units + will be 'hours since' each start date and the time steps are assumed to be + equally spaced. +} +\item{\code{single_file = FALSE}}{ + The data array is subset and stored into multiple files. Each file + contains the data subset for each start date, variable and dataset. Files + with different variables and Datasets are stored in separated directories + within the following directory tree: destination/Dataset/variable/. + The name of each file will be: + __.nc. +} } \description{ This function allows to save a data array with metadata into a @@ -111,6 +122,7 @@ SaveExp(data = data, destination = destination, coords = coords, metadata = metadata, single_file = TRUE, ftime_dim = 'ftime', var_dim = NULL) } + } \author{ Perez-Zanon Nuria, \email{nuria.perez@bsc.es} diff --git a/tests/testthat/test-CST_Subset.R b/tests/testthat/test-CST_Subset.R index 27e3cbef5ff3f21336e6aef91ec68f9e3d633136..313673ba495427abc62e029b930dc858642763fc 100644 --- a/tests/testthat/test-CST_Subset.R +++ b/tests/testthat/test-CST_Subset.R @@ -2,6 +2,10 @@ context("CSTools::CST_Subset tests") ############################################## +library(startR) + +############################################## + test_that("1. Input checks: CST_Subset", { # Check that x is s2dv_cube expect_error( @@ -58,7 +62,7 @@ test_that("2. Output checks: CST_Subset", { c("lon", "prlr") ) expect_equal( - names(res1$attrs$Datasets), + res1$attrs$Datasets, c("exp1") ) # Check 'dat_dim' @@ -71,14 +75,12 @@ test_that("2. Output checks: CST_Subset", { res5 <- CST_Subset(lonlat_prec, along = c('dataset'), indices = list(1), drop = 'selected', dat_dim = 'dataset') expect_equal( - names(res2$attrs$Datasets), - names(res3$attrs$Datasets), - "exp1" + res2$attrs$Datasets, + res3$attrs$Datasets ) expect_equal( length(res4$attrs$Datasets), - length(res5$attrs$Datasets), - 0 + length(res5$attrs$Datasets) ) # Check 'Dates' res6 <- CST_Subset(lonlat_prec, along = c('sdate', 'ftime'), diff --git a/tests/testthat/test-as.s2dv_cube.R b/tests/testthat/test-as.s2dv_cube.R index 36a24738f0d791b2b031af637b5b5f4db024340b..5d6303d583b53f61435a57d57917cc8eccbbf92a 100644 --- a/tests/testthat/test-as.s2dv_cube.R +++ b/tests/testthat/test-as.s2dv_cube.R @@ -19,125 +19,125 @@ test_that("1. Input checks", { ############################################## -# test_that("2. Tests from Load()", { -# startDates <- c('20001101', '20011101') -# suppressWarnings( -# ob1 <- Load(var = 'tas', exp = 'system5c3s', -# nmember = 2, sdates = startDates, -# leadtimemax = 3, latmin = 30, latmax = 35, -# lonmin = 10, lonmax = 20, output = 'lonlat') -# ) -# res1 <- as.s2dv_cube(ob1) +test_that("2. Tests from Load()", { + startDates <- c('20001101', '20011101') + suppressWarnings( + ob1 <- Load(var = 'tas', exp = 'system5c3s', + nmember = 2, sdates = startDates, + leadtimemax = 3, latmin = 30, latmax = 35, + lonmin = 10, lonmax = 20, output = 'lonlat') + ) + res1 <- as.s2dv_cube(ob1) -# # dimensions -# expect_equal( -# dim(res1$data), -# c(dataset = 1, member = 2, sdate = 2, ftime = 3, lat = 6, lon = 11) -# ) -# # elements -# expect_equal( -# names(res1), -# c("data", "dims", "coords", "attrs") -# ) -# expect_equal( -# names(res1$attrs), -# c("Variable", "Datasets", "Dates", "when", "source_files", -# "not_found_files", "load_parameters") -# ) -# # coordinates -# expect_equal( -# attributes(res1$coords$sdate), -# list(indices = FALSE) -# ) -# expect_equal( -# attributes(res1$coords$ftime), -# list(indices = TRUE) -# ) -# # Dates -# expect_equal( -# dim(res1$attrs$Dates), -# c(ftime = 3, sdate = 2) -# ) -# }) + # dimensions + expect_equal( + dim(res1$data), + c(dataset = 1, member = 2, sdate = 2, ftime = 3, lat = 6, lon = 11) + ) + # elements + expect_equal( + names(res1), + c("data", "dims", "coords", "attrs") + ) + expect_equal( + names(res1$attrs), + c("Variable", "Datasets", "Dates", "when", "source_files", + "not_found_files", "load_parameters") + ) + # coordinates + expect_equal( + attributes(res1$coords$sdate), + list(indices = FALSE) + ) + expect_equal( + attributes(res1$coords$ftime), + list(indices = TRUE) + ) + # Dates + expect_equal( + dim(res1$attrs$Dates), + c(ftime = 3, sdate = 2) + ) +}) ############################################## -# test_that("3. Tests from Load()", { -# obs_path <- list(name = "ERA5", -# path = "/esarchive/recon/ecmwf/era5/$STORE_FREQ$_mean/$VAR_NAME$_f1h/$VAR_NAME$_$YEAR$$MONTH$.nc") -# ob2 <- Load(var = 'windagl100', obs = list(obs_path), -# sdates = '20180301', nmember = 1, -# leadtimemin = 1, leadtimemax = 1, -# storefreq = "monthly", sampleperiod = 1, -# latmin = 36, latmax = 38, lonmin = 0, lonmax = 4, -# output = 'lonlat', nprocs = 1, grid = 'r360x181') +test_that("3. Tests from Load()", { + obs_path <- list(name = "ERA5", + path = "/esarchive/recon/ecmwf/era5/$STORE_FREQ$_mean/$VAR_NAME$_f1h/$VAR_NAME$_$YEAR$$MONTH$.nc") + ob2 <- Load(var = 'windagl100', obs = list(obs_path), + sdates = '20180301', nmember = 1, + leadtimemin = 1, leadtimemax = 1, + storefreq = "monthly", sampleperiod = 1, + latmin = 36, latmax = 38, lonmin = 0, lonmax = 4, + output = 'lonlat', nprocs = 1, grid = 'r360x181') -# res2 <- as.s2dv_cube(ob2) + res2 <- as.s2dv_cube(ob2) -# # dimensions -# expect_equal( -# dim(res2$data), -# c(dataset = 1, member = 1, sdate = 1, ftime = 1, lat = 3, lon = 5) -# ) -# # elements -# expect_equal( -# names(res2$attrs), -# c("Variable", "Datasets", "Dates", "when", "source_files", -# "not_found_files", "load_parameters") -# ) -# # coordinates -# expect_equal( -# attributes(res2$coords$sdate), -# list(indices = FALSE) -# ) -# expect_equal( -# unlist(res2$coords)[1:4], -# c(dataset = "1", member = "1", sdate = "20180301", ftime = "1") -# ) -# # Dates -# expect_equal( -# dim(res2$attrs$Dates), -# c(ftime = 1, sdate = 1) -# ) -# }) + # dimensions + expect_equal( + dim(res2$data), + c(dataset = 1, member = 1, sdate = 1, ftime = 1, lat = 3, lon = 5) + ) + # elements + expect_equal( + names(res2$attrs), + c("Variable", "Datasets", "Dates", "when", "source_files", + "not_found_files", "load_parameters") + ) + # coordinates + expect_equal( + attributes(res2$coords$sdate), + list(indices = FALSE) + ) + expect_equal( + unlist(res2$coords)[1:4], + c(dataset = "ERA5", member = "1", sdate = "20180301", ftime = "1") + ) + # Dates + expect_equal( + dim(res2$attrs$Dates), + c(ftime = 1, sdate = 1) + ) +}) ############################################## -# test_that("4. Tests from Load()", { -# exp <- list(name = 'ecmwfS5', -# path = "/esarchive/exp/ecmwf/system5c3s/$STORE_FREQ$_mean/$VAR_NAME$_s0-24h/$VAR_NAME$_$START_DATE$.nc") -# obs <- list(name = 'era5', -# path = '/esarchive/recon/ecmwf/era5/$STORE_FREQ$_mean/$VAR_NAME$_f1h-r1440x721cds/$VAR_NAME$_$YEAR$$MONTH$.nc') -# suppressWarnings( -# ob3 <- Load(var = 'prlr', exp = list(exp), obs = list(obs), -# sdates = paste0(1993:1995, '1101'), nmember = 1, -# storefreq = "monthly", sampleperiod = 1, -# latmin = 42, latmax = 45, lonmin = 4, lonmax = 6, -# output = 'lonlat', nprocs = 1) -# ) -# expect_warning( -# as.s2dv_cube(ob3), -# "The output is a list of two 's2dv_cube' objects corresponding to 'exp' and 'obs'." -# ) -# suppressWarnings( -# res3 <- as.s2dv_cube(ob3) -# ) +test_that("4. Tests from Load()", { + exp <- list(name = 'ecmwfS5', + path = "/esarchive/exp/ecmwf/system5c3s/$STORE_FREQ$_mean/$VAR_NAME$_s0-24h/$VAR_NAME$_$START_DATE$.nc") + obs <- list(name = 'era5', + path = '/esarchive/recon/ecmwf/era5/$STORE_FREQ$_mean/$VAR_NAME$_f1h-r1440x721cds/$VAR_NAME$_$YEAR$$MONTH$.nc') + suppressWarnings( + ob3 <- Load(var = 'prlr', exp = list(exp), obs = list(obs), + sdates = paste0(1993:1995, '1101'), nmember = 1, + storefreq = "monthly", sampleperiod = 1, + latmin = 42, latmax = 45, lonmin = 4, lonmax = 6, + output = 'lonlat', nprocs = 1) + ) + expect_warning( + as.s2dv_cube(ob3), + "The output is a list of two 's2dv_cube' objects corresponding to 'exp' and 'obs'." + ) + suppressWarnings( + res3 <- as.s2dv_cube(ob3) + ) -# # dimensions -# expect_equal( -# dim(res3[[1]]$data), -# c(dataset = 1, member = 1, sdate = 3, ftime = 8, lat = 4, lon = 3) -# ) -# expect_equal( -# unlist(res3[[1]]$coords)[1:4], -# c(dataset = "1", member = "1", sdate1 = "19931101", sdate2 = "19941101") -# ) -# # Dates -# expect_equal( -# dim(res3[[1]]$attrs$Dates), -# dim(res3[[2]]$attrs$Dates) -# ) -# }) + # dimensions + expect_equal( + dim(res3[[1]]$data), + c(dataset = 1, member = 1, sdate = 3, ftime = 8, lat = 4, lon = 3) + ) + expect_equal( + unlist(res3[[1]]$coords)[1:4], + c(dataset = "ecmwfS5", member = "1", sdate1 = "19931101", sdate2 = "19941101") + ) + # Dates + expect_equal( + dim(res3[[1]]$attrs$Dates), + dim(res3[[2]]$attrs$Dates) + ) +}) ##############################################