From 08709b8036fd353f882b5656435e0224949616c4 Mon Sep 17 00:00:00 2001 From: PALOMA TRASCASA CASTRO Date: Tue, 16 Sep 2025 15:25:10 +0200 Subject: [PATCH 1/2] fixed cron and scripts, now working to copy and rename subseasonal and seasonal files into Bigpredidata's ftp --- ...erational_protocol_bigpredidata_boreas.txt | 6 +-- operationals/bigpredidata/seasonal_copy.sh | 37 ++++++++++++++++ operationals/bigpredidata/subseasonal_copy.sh | 42 +++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100755 operationals/bigpredidata/seasonal_copy.sh create mode 100755 operationals/bigpredidata/subseasonal_copy.sh diff --git a/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt index 78182ad9..d2fa8104 100644 --- a/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt +++ b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt @@ -11,12 +11,12 @@ The files go in the FTP folder, where we share data with RawData: - /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/subseasonal_copy.sh # This is what the crontab -e file should look like: + # Subseasonal: run every day at 10:00 -0 10 * * * ~/subseasonal_copy.sh >> ~/subseasonal.log 2>&1 +0 10 * * * /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/subseasonal_copy.sh >> /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/subseasonal.log 2>&1 # Seasonal: run every Wednesday at 10:00 -0 10 * * 3 ~/seasonal_copy.sh >> ~/seasonal.log 2>&1 - +0 10 * * 3 /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/seasonal_copy.sh >> /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/seasonal.log 2>&1 Seasonal • Percentiles and skill files are already in the folder, so just the probability files should be updated. diff --git a/operationals/bigpredidata/seasonal_copy.sh b/operationals/bigpredidata/seasonal_copy.sh new file mode 100755 index 00000000..64e3d6fb --- /dev/null +++ b/operationals/bigpredidata/seasonal_copy.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -euo pipefail + +outdir="/esarchive/scratch/esarchive_ftp/bigpredidata/test_rawdata/" +var_list=("prlr" "rsds" "sfcWind" "tas" "tasmax" "tasmin") + +# Seasonal sdate = first day of current month +sdate=$(date +%Y%m01) +month=$(date +%m) + +echo "========== Seasonal run for $sdate ==========" + +for var in "${var_list[@]}"; do + echo "---- Processing variable: $var ----" + + seasonal_probs_file="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_oper_${sdate}/outputs/Calibration/Iberia/ECMWF-SEAS51/bias-monthly_mean/${var}/${sdate}/${var}-probs_${sdate}.nc" + + echo "Looking for: $seasonal_probs_file" + if [[ -f "$seasonal_probs_file" ]]; then + base=$(basename "$seasonal_probs_file" .nc) # remove .nc extension + newfile="${outdir}scorecards_ECMWF-SEAS51_ERA5_${base}_1993-2016_s${month}.nc" + echo " → Found. Copying to: $newfile" + cp -v "$seasonal_probs_file" "$newfile" + if [[ -f "$newfile" ]]; then + echo " ✔ Confirmed copied: $newfile" + else + echo " ✖ Copy failed: $newfile" + fi + else + echo " ✖ File not found: $seasonal_probs_file" + fi + + echo +done + +echo "✅ Seasonal copy finished for $sdate" + diff --git a/operationals/bigpredidata/subseasonal_copy.sh b/operationals/bigpredidata/subseasonal_copy.sh new file mode 100755 index 00000000..593d2f7a --- /dev/null +++ b/operationals/bigpredidata/subseasonal_copy.sh @@ -0,0 +1,42 @@ +#!/bin/bash +export PATH=/usr/local/bin:/usr/bin:/bin +set -euo pipefail + +outdir="/esarchive/scratch/esarchive_ftp/bigpredidata/test_rawdata/" +var_list=("prlr" "rsds" "sfcWind" "tas" "tasmax" "tasmin") + +# Get last Thursday +today=$(date +%Y-%m-%d) +last_thursday=$(date -d "last thursday" +%Y%m%d) + +for var in "${var_list[@]}"; do + # Source directories + subseasonal_probs_dir="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_subseasonal_oper_${last_thursday}/outputs/Calibration/Iberia/NCEP-CFSv2/bias-weekly_mean/${var}/${last_thursday}/${var}-probs_${last_thursday}.nc" + subseasonal_perc_dir="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_subseasonal_oper_${last_thursday}/outputs/Calibration/Iberia/NCEP-CFSv2/bias-weekly_mean/${var}/${last_thursday}/${var}-percentiles_week*.nc" + subseasonal_skill_dir="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_subseasonal_oper_${last_thursday}/outputs/Skill/Iberia/NCEP-CFSv2/bias-weekly_mean/${var}/${last_thursday}/${var}-skill_week*.nc" + + echo "========== Processing variable: $var ==========" + +# Copy and rename with checks + for file in $subseasonal_probs_dir $subseasonal_perc_dir $subseasonal_skill_dir; do + for f in $file; do + echo "Looking for: $f" + if [[ -f "$f" ]]; then + base=$(basename "$f") + newfile="${outdir}NCEP-CFSv2_${base}" + echo " → Found. Copying to: $newfile" + cp -v "$f" "$newfile" + if [[ -f "$newfile" ]]; then + echo " ✔ Confirmed copied: $newfile" + else + echo " ✖ Copy failed: $newfile" + fi + else + echo " ✖ File not found: $f" + fi + done + done + echo +done + +echo "✅ All variables processed." -- GitLab From c2ddc8d30ed1ce1048bc7c1c112513917a881b76 Mon Sep 17 00:00:00 2001 From: Paloma Trascasa-Castro Date: Tue, 16 Sep 2025 15:40:58 +0200 Subject: [PATCH 2/2] Update operational_protocol_bigpredidata_boreas.txt --- .../bigpredidata/operational_protocol_bigpredidata_boreas.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt index d2fa8104..605fda73 100644 --- a/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt +++ b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt @@ -1,4 +1,4 @@ -BigPrediData & BOREAS OPERATIONALS +BigPrediData & BOREAS OPERATIONALS 1) Files -- GitLab