diff --git a/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt index 78182ad915bee6232bf6699aedec0e460e287f01..605fda73e672076bcce5acdb6531c6f283f520b6 100644 --- a/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt +++ b/operationals/bigpredidata/operational_protocol_bigpredidata_boreas.txt @@ -1,4 +1,4 @@ -BigPrediData & BOREAS OPERATIONALS +BigPrediData & BOREAS OPERATIONALS 1) Files @@ -11,12 +11,12 @@ The files go in the FTP folder, where we share data with RawData: - /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/subseasonal_copy.sh # This is what the crontab -e file should look like: + # Subseasonal: run every day at 10:00 -0 10 * * * ~/subseasonal_copy.sh >> ~/subseasonal.log 2>&1 +0 10 * * * /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/subseasonal_copy.sh >> /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/subseasonal.log 2>&1 # Seasonal: run every Wednesday at 10:00 -0 10 * * 3 ~/seasonal_copy.sh >> ~/seasonal.log 2>&1 - +0 10 * * 3 /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/seasonal_copy.sh >> /esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/seasonal.log 2>&1 Seasonal • Percentiles and skill files are already in the folder, so just the probability files should be updated. diff --git a/operationals/bigpredidata/seasonal_copy.sh b/operationals/bigpredidata/seasonal_copy.sh new file mode 100755 index 0000000000000000000000000000000000000000..64e3d6fb4825e72c6c89daaa82e1dd154c259b3d --- /dev/null +++ b/operationals/bigpredidata/seasonal_copy.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -euo pipefail + +outdir="/esarchive/scratch/esarchive_ftp/bigpredidata/test_rawdata/" +var_list=("prlr" "rsds" "sfcWind" "tas" "tasmax" "tasmin") + +# Seasonal sdate = first day of current month +sdate=$(date +%Y%m01) +month=$(date +%m) + +echo "========== Seasonal run for $sdate ==========" + +for var in "${var_list[@]}"; do + echo "---- Processing variable: $var ----" + + seasonal_probs_file="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_oper_${sdate}/outputs/Calibration/Iberia/ECMWF-SEAS51/bias-monthly_mean/${var}/${sdate}/${var}-probs_${sdate}.nc" + + echo "Looking for: $seasonal_probs_file" + if [[ -f "$seasonal_probs_file" ]]; then + base=$(basename "$seasonal_probs_file" .nc) # remove .nc extension + newfile="${outdir}scorecards_ECMWF-SEAS51_ERA5_${base}_1993-2016_s${month}.nc" + echo " → Found. Copying to: $newfile" + cp -v "$seasonal_probs_file" "$newfile" + if [[ -f "$newfile" ]]; then + echo " ✔ Confirmed copied: $newfile" + else + echo " ✖ Copy failed: $newfile" + fi + else + echo " ✖ File not found: $seasonal_probs_file" + fi + + echo +done + +echo "✅ Seasonal copy finished for $sdate" + diff --git a/operationals/bigpredidata/subseasonal_copy.sh b/operationals/bigpredidata/subseasonal_copy.sh new file mode 100755 index 0000000000000000000000000000000000000000..593d2f7af46101d506d717c8511dc4d15883b31a --- /dev/null +++ b/operationals/bigpredidata/subseasonal_copy.sh @@ -0,0 +1,42 @@ +#!/bin/bash +export PATH=/usr/local/bin:/usr/bin:/bin +set -euo pipefail + +outdir="/esarchive/scratch/esarchive_ftp/bigpredidata/test_rawdata/" +var_list=("prlr" "rsds" "sfcWind" "tas" "tasmax" "tasmin") + +# Get last Thursday +today=$(date +%Y-%m-%d) +last_thursday=$(date -d "last thursday" +%Y%m%d) + +for var in "${var_list[@]}"; do + # Source directories + subseasonal_probs_dir="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_subseasonal_oper_${last_thursday}/outputs/Calibration/Iberia/NCEP-CFSv2/bias-weekly_mean/${var}/${last_thursday}/${var}-probs_${last_thursday}.nc" + subseasonal_perc_dir="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_subseasonal_oper_${last_thursday}/outputs/Calibration/Iberia/NCEP-CFSv2/bias-weekly_mean/${var}/${last_thursday}/${var}-percentiles_week*.nc" + subseasonal_skill_dir="/esarchive/scratch/ptrascas/R/dev-test_bigpredidata/sunset/recipe_bigpredidata_oper_subseasonal_oper_${last_thursday}/outputs/Skill/Iberia/NCEP-CFSv2/bias-weekly_mean/${var}/${last_thursday}/${var}-skill_week*.nc" + + echo "========== Processing variable: $var ==========" + +# Copy and rename with checks + for file in $subseasonal_probs_dir $subseasonal_perc_dir $subseasonal_skill_dir; do + for f in $file; do + echo "Looking for: $f" + if [[ -f "$f" ]]; then + base=$(basename "$f") + newfile="${outdir}NCEP-CFSv2_${base}" + echo " → Found. Copying to: $newfile" + cp -v "$f" "$newfile" + if [[ -f "$newfile" ]]; then + echo " ✔ Confirmed copied: $newfile" + else + echo " ✖ Copy failed: $newfile" + fi + else + echo " ✖ File not found: $f" + fi + done + done + echo +done + +echo "✅ All variables processed."