diff --git a/.gitignore b/.gitignore index 263c4e640a4ffe3bd13bf6a14f80c37553954d4d..f19c4813d6876e803e09a3dceab2db8a51d33df9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ ecsbatch.log* modules/Loading/testing_recipes/recipe_decadal_calendartest.yml modules/Loading/testing_recipes/recipe_decadal_daily_calendartest.yml conf/vitigeoss-vars-dict.yml +operationals/bigpredidata/recipe_bigpredidata_oper_oper_* +operationals/bigpredidata/recipe_bigpredidata_oper_subseasonal_oper_* +*.log diff --git a/modules/Visualization/output_size.yml b/modules/Visualization/output_size.yml index 74e3f1548881959973970e1220de103395798d3a..4b77a6ff80c482bfce3432bc78b91be50bf5f21e 100644 --- a/modules/Visualization/output_size.yml +++ b/modules/Visualization/output_size.yml @@ -88,6 +88,9 @@ region: #units inches dot_symbol: 16 font.main: 1 colNA: "white" + coast_color: "#00000000" + coast_width: 0.0001 + country.borders: FALSE forecast_map: width: 8 height: 7.5 @@ -111,6 +114,8 @@ region: #units inches bar_extra_margin: !expr c(2.5, 1, 0.5, 1) # colorbar proportions colNA: "white" col_mask: 'white' + coast_color: '#00000000' + coast_width: 0.0001 country.borders: FALSE extreme_probabilities: width: 8 @@ -125,6 +130,8 @@ region: #units inches dot_size: 1 font.main: 1 colNA: "white" + coast_color: '#00000000' + coast_width: 0.0001 country.borders: FALSE multipanel: forecast_map: diff --git a/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt new file mode 100644 index 0000000000000000000000000000000000000000..c8834869e1a569a892841178856a3944380f9038 --- /dev/null +++ b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt @@ -0,0 +1,36 @@ +BigPrediData & BOREAS OPERATIONALS + +1) Files + +The files go in the FTP folder, where we share data with RawData: + +/esarchive/scratch/esarchive_ftp/bigpredidata/test_rawdata/ + +Seasonal + • Percentiles and skill files are already in the folder, so just the probability files should be updated. + • Files should have this name: “scorecards_ECMWF-SEAS51_ERA5_variable-probs_sdate_1993-2016_smonth.nc” , for example: “scorecards_ECMWF-SEAS51_ERA5_prlr-probs_20250701_1993-2016_s07.nc” + +Subseasonal + • Percentiles, skill and probabilities need to be updated weekly to this folder. + • Probability files should have this name: NCEP-CFSv2_variable-probs_sdate.nc, for example “NCEP-CFSv2_tas-probs_20250703.nc”. + • Skill and percentile files should have the week number instead of the sdate, like in here: “NCEP-CFSv2_prlr-skill_week27.nc” + +2) Figures +The figure files appear in my output directory of SUNSET: /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset, and should be copied in this other directory: “/esarchive/scratch/ptrascas/bigpredidata/operational/figures/” +, where I keep the figures for both the BigPrediData and BOREAS shiny. +In this folder the files should follow this exact structure so the shinies can read them: +horizon_region_variable_forecast_most_likely_tercile-sdate_rpssmask_ftime.png, for example: subseasonal_Iberia_tasmin_forecast_most_likely_tercile-20250724_rpssmask_ft01.png + +3) Shiny +The shiny apps for BOREAS and BigPrediData are in esarchive. After edit them, we must move them to the shiny server: + + +esarchive +Shiny server +BigPrediData +/esarchive/scratch/ptrascas/bigpredidata/operational/app.R +/srv/shiny-server/es/ptrascas_bigpredidata_oper/app.R +BOREAS +/esarchive/scratch/ptrascas/R/ptrascas_scripts/shiny_BOREAS/app.R +/srv/shiny-server/es/ptrascas_BOREAS_fqa/app.R + diff --git a/operationals/bigpredidata/recipe_bigpredidata_oper.yml b/operationals/bigpredidata/recipe_bigpredidata_oper.yml index 4e45d3d0f1ec5c05891de3070e11fac47cef743b..47b4e8a3da045dd9cff46ef7d7887619ed3fa986 100644 --- a/operationals/bigpredidata/recipe_bigpredidata_oper.yml +++ b/operationals/bigpredidata/recipe_bigpredidata_oper.yml @@ -70,9 +70,10 @@ Analysis: brks_max: 95 NA_color: white multi_panel: no - dots: no - mask_terciles: yes - shapefile: shapefiles/bigpredidata/recintos_provinciales_inspire_peninbal_etrs89.shp + dots: yes + mask_terciles: no + shapefile: shapefiles/bigpredidata/merged_countries_spain.shp + #shapefiles/bigpredidata/recintos_provinciales_inspire_peninbal_etrs89.shp file_format: PNG # Final file format of the plots. Formats available: PNG, JPG, JPEG, EPS. Defaults to PDF. ncores: 12 # Optional, int: number of cores, defaults to 1 remove_NAs: TRUE # Optional, bool: Whether NAs are removed, defaults to FALSE diff --git a/operationals/bigpredidata/recipe_bigpredidata_oper_subseasonal.yml b/operationals/bigpredidata/recipe_bigpredidata_oper_subseasonal.yml index 83e9f86c4449f3fb8d3caba156a1a8160b4cf002..b06a0b2e61aed8afc3c4d5d03261f9a129648f21 100755 --- a/operationals/bigpredidata/recipe_bigpredidata_oper_subseasonal.yml +++ b/operationals/bigpredidata/recipe_bigpredidata_oper_subseasonal.yml @@ -63,9 +63,9 @@ Analysis: brks_max: 95 NA_color: white multi_panel: no - dots: no - mask_terciles: yes - shapefile: shapefiles/bigpredidata/recintos_provinciales_inspire_peninbal_etrs89.shp + dots: yes + mask_terciles: no + shapefile: shapefiles/bigpredidata/merged_countries_spain.shp file_format: PNG # Final file format of the plots. Formats available: PNG, JPG, JPEG, EPS. Defaults to PDF. ncores: 12 # Optional, int: number of cores, defaults to 1 remove_NAs: TRUE # Optional, bool: Whether NAs are removed, defaults to FALSE diff --git a/recipes/recipe_test.yml b/recipes/recipe_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..46b99e95e34e950a4f1fe662ca8d217439a80a41 --- /dev/null +++ b/recipes/recipe_test.yml @@ -0,0 +1,113 @@ +Description: + Author: Paloma Trascasa-Castro + Info: Fqa seasonal forecasts - multivariable, multimodel, Iberia and Europe, all calibration methods, anomalies and absolute, cross validation. + +Analysis: + Horizon: seasonal # Mandatory, str: either subseasonal, seasonal, or decadal + Variables: + # - {name: tasmin, freq: monthly_mean, units: C} + # - {name: tasmax, freq: monthly_mean, units: C} + - {name: tas, freq: monthly_mean, units: C} + # - {name: prlr, freq: monthly_mean, units: mm, flux: no} + # - {name: sfcWind, freq: monthly_mean} + # - {name: rsds, freq: monthly_mean} + Datasets: + System: + - {name: ECMWF-SEAS5.1} + # Add Meteo-France System 9 when available + # - {name: DWD-GCFS2.1} + # - {name: CMCC-SPS3.5} + # - {name: JMA-MRI-CPS3} + # - {name: ECC-GEM5.2-NEMO}i # Is this the most recent one? + # - {name: UK-MetOffice-Glosea603} + # - {name: NCEP-CFSv2} + # Add BoM ACCESS-S2 when available + + Multimodel: no + # execute: both + # approach: pooled + # createFrom: Calibration # if "absolute" poner "Calibration" + Reference: + - {name: ERA5} # Mandatory, str: Reference codename. See docu. + Time: + sdate: + # - '0101' + # - '0201' + # - '0301' + # - '0401' + # - '0501' + - '0601' + # - '0701' + # - '0801' + # - '0901' + # - '1001' + # - '1101' + # - '1201' + fcst_year: 2025 # Optional, int: Forecast year 'YYYY' + hcst_start: '1993' # Mandatory, int: Hindcast start year 'YYYY' + hcst_end: '2016' # Mandatory, int: Hindcast end year 'YYYY' + ftime_min: 1 # Mandatory, int: First leadtime time step in months + ftime_max: 6 # Mandatory, int: Last leadtime time step in months + Region: + - {name: "Iberia", latmin: 36, latmax: 44, lonmin: -10, lonmax: 5} + # - {name: "EU", latmin: 20, latmax: 80, lonmin: -20, lonmax: 40} + Regrid: + method: bilinear # Mandatory, str: Interpolation method. See docu. + type: "to_system" + #type: /esarchive/scratch/nmilders/gitlab/git_clones/auto-s2s/conf/grid_description.txt #'r360x180' # Mandatory, str: to_system, to_reference, or CDO-accepted grid. + Workflow: + Anomalies: + compute: no + cross_validation: no + save: none + Calibration: + method: bias # Mandatory, str: bias, evmos, mse_min, crps_min, rpc_based + cross_validation: yes + save: none + Skill: + metric: rpss #mean_bias EnsCorr rps rpss crps crpss #add briers + save: none + cross_validation: yes + Probabilities: + percentiles: #[[1/3, 2/3]] + Terciles: [1/3, 2/3] + P10: [1/10] + P90: [9/10] + save: "all" + Indicators: + index: no + Visualization: + plots: most_likely_terciles, extreme_probabilities + brks_min: 5 + brks_max: 95 + NA_color: white + multi_panel: no + dots: no + mask_terciles: no +# shapefile: recintos_provinciales_inspire_peninbal_etrs89.shp + file_format: PNG # Final file format of the plots. Formats available: PNG, JPG, JPEG, EPS. Defaults to PDF. + ncores: 12 # Optional, int: number of cores, defaults to 1 + remove_NAs: TRUE # Optional, bool: Whether NAs are removed, defaults to FALSE + Output_format: S2S4E + logo: yes +Run: + Loglevel: INFO + Terminal: yes + filesystem: gpfs + output_dir: /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset # replace with the directory where you want to save the outputs + code_dir: /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/sunset # replace with the directory where your code is + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: script_boreas_fqa.R # replace with the path to your script + expid: a99d # replace with your EXPID + hpc_user: bsc032413 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 24 + custom_directives: # ['#SBATCH --exclusive'] + platform: amd + email_notifications: no # enable/disable email notifications. Change it if you want to. + email_address: paloma.trascasa@bsc.es # replace with your email address + notify_completed: yes # notify me by email when a job finishes + notify_failed: no # notify me by email when a job fails + diff --git a/scripts/script_boreas_fqa.R b/scripts/script_boreas_fqa.R new file mode 100644 index 0000000000000000000000000000000000000000..be597fa0b9177a1d680597c751dd345a91ea1368 --- /dev/null +++ b/scripts/script_boreas_fqa.R @@ -0,0 +1,45 @@ +source("modules/Loading/Loading.R") +source("modules/Saving/Saving.R") +source("modules/Units/Units.R") +source("modules/Visualization/Visualization.R") +source("modules/Aggregation/Aggregation.R") + +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] +#recipe_file <- "/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/sunset/recipes/recipe_bigpredidata_oper_test_sfcWind.yml" +#recipe_file <- "/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/sunset/recipes/recipe_bigpredidata_oper.yml" +recipe <- read_atomic_recipe(recipe_file) +#recipe <- prepare_outputs(recipe_file) +# Load datasets +data <- Loading(recipe) +data <- Units(recipe, data) +data_summary(data$hcst, recipe) +data_summary(data$obs, recipe) +data_summary(data$fcst, recipe) + +if (!is.null(data$fcst)) { + data_summary(data$fcst, recipe) +} + +#source("modules/Crossval/Crossval_Calibration.R") +#res <- Crossval_Calibration(recipe = recipe, data = data) + +# Save calibrated hindcast, forecast and probabilities + +source("modules/Crossval/Crossval_metrics.R") +skill_metrics <- Crossval_metrics(recipe = recipe, data_crossval = res, + fair = FALSE, nmemb = NULL, nmemb_ref = NULL) + + + +if (!is.null(data$fcst)) { + # Required to plot a forecast: + data$fcst <- res$fcst + Visualization(recipe = recipe, data = data, skill_metrics = skill_metrics, + significance = TRUE, probabilities = res$probs) +} else { + Visualization(recipe = recipe, data = data, skill_metrics = skill_metrics, + significance = TRUE) +} + + diff --git a/shapefiles/bigpredidata/merged_continents_spain.dbf b/shapefiles/bigpredidata/merged_continents_spain.dbf new file mode 100644 index 0000000000000000000000000000000000000000..3cd4399d4b3bcfc1eb83135a1896e8e17255710b Binary files /dev/null and b/shapefiles/bigpredidata/merged_continents_spain.dbf differ diff --git a/shapefiles/bigpredidata/merged_continents_spain.prj b/shapefiles/bigpredidata/merged_continents_spain.prj new file mode 100644 index 0000000000000000000000000000000000000000..f45cbadf0074d8b7b2669559a93bc50bb95f82d4 --- /dev/null +++ b/shapefiles/bigpredidata/merged_continents_spain.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] \ No newline at end of file diff --git a/shapefiles/bigpredidata/merged_continents_spain.shp b/shapefiles/bigpredidata/merged_continents_spain.shp new file mode 100644 index 0000000000000000000000000000000000000000..6e8f7f602dc57339a1416b1c25ded10216962418 Binary files /dev/null and b/shapefiles/bigpredidata/merged_continents_spain.shp differ diff --git a/shapefiles/bigpredidata/merged_continents_spain.shx b/shapefiles/bigpredidata/merged_continents_spain.shx new file mode 100644 index 0000000000000000000000000000000000000000..4cef9625dcd4b3510c203229ce717d624bbcb109 Binary files /dev/null and b/shapefiles/bigpredidata/merged_continents_spain.shx differ diff --git a/shapefiles/bigpredidata/merged_countries_spain.dbf b/shapefiles/bigpredidata/merged_countries_spain.dbf new file mode 100644 index 0000000000000000000000000000000000000000..75672d11ca9d980122c43c1e790ddff838c8d92e Binary files /dev/null and b/shapefiles/bigpredidata/merged_countries_spain.dbf differ diff --git a/shapefiles/bigpredidata/merged_countries_spain.prj b/shapefiles/bigpredidata/merged_countries_spain.prj new file mode 100644 index 0000000000000000000000000000000000000000..f45cbadf0074d8b7b2669559a93bc50bb95f82d4 --- /dev/null +++ b/shapefiles/bigpredidata/merged_countries_spain.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] \ No newline at end of file diff --git a/shapefiles/bigpredidata/merged_countries_spain.shp b/shapefiles/bigpredidata/merged_countries_spain.shp new file mode 100644 index 0000000000000000000000000000000000000000..98cba58c7b867cc32ab3a22e706e58a05a3ee476 Binary files /dev/null and b/shapefiles/bigpredidata/merged_countries_spain.shp differ diff --git a/shapefiles/bigpredidata/merged_countries_spain.shx b/shapefiles/bigpredidata/merged_countries_spain.shx new file mode 100644 index 0000000000000000000000000000000000000000..27984d3e0abee7c7a1e7f6dc5ddf9e35e6b9912f Binary files /dev/null and b/shapefiles/bigpredidata/merged_countries_spain.shx differ