source ~/.bashrc

# for each variable located in  /shared/cds_downloads/${dataset_type}/${dataset}
# 1- build the zip name that should be in the results folder
# for instance seasonal-monthly-single-levels/10m_u_component_of_wind would be
# seasonal-monthly-single-levels-10m_u_component_of_wind

# 2 - look in the QAR folder (/shared/QAR_dqc/results/) if there any matching zip file and for each of these
# remove the dataset-varname from the name itself and also the extension so we get the sistem for which the QAR existsdd
# then inform or remove the file in the /shared/cds_downloads/ for such system

# sample usage:
# ./free_space_generated_qar.sh seasonal seasonal-monthly-pressure-levels | tee /data/julian/seasonal-monthly-pressure-levels.txt

source ~/.bashrc

# Filter
delete_data="false"
if [[ -z "$1" ]] && [[ -z "$2" ]] ; then
    echo "No filter arguments supplied"
    exit 1
fi

dataset_type=$1
dataset=$2
if [ "$#" -eq 3 ]; then
    delete_data=$3
fi

path_to_export=echo $(pwd)
if [ "$#" -eq 4 ]; then
    path_to_export=$4
fi



DATA_DIR="/shared/cds_downloads"
QAR_RES_DIR="/shared/QAR_dqc/results"

echo "checking dir "${DATA_DIR}/${dataset_type}/${dataset}


for variable in `ls -1 ${DATA_DIR}/${dataset_type}/${dataset}*`
do
  if [[ ${dataset_type} == "seasonal" ]] ; then
     echo "processing:"  $(basename ${variable})
     qar_filename_exp=${dataset}-$(basename ${variable})*.zip
     echo ${qar_filename_exp}
     for qar in `ls -1 ${QAR_RES_DIR}/${qar_filename_exp}`
     do
         qar_filename=$(basename ${qar})
         prefix=${dataset}-${variable}-
         datafile=${qar_filename#$( echo ${prefix})}
         datafile=${datafile%.zip}
        echo "QAR  ${qar_filename} exists"
         if [[ ${delete_data} == "true" ]] ; then
           echo "deleting "${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*${datafile}*.grib
           rm ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*${datafile}*.grib
           rm ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*${datafile}*.nc
           rm ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*${datafile}*.csv
           rm ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*${datafile}*.idx
         else
            echo "the following files can be removed: " ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/ "for system ${datafile}"
            ls  ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*${datafile}* > ${path_to_export}/files-to-be-removed-${dataset}-${variable}-${datafile}.txt
         fi
     done
  elif [[ ${dataset_type} == "reanalysis" ]] ; then
     echo "processing:"  $(basename ${variable})
     qar_filename_exp=${dataset}-${variable}.zip
     if [[ -f ${QAR_RES_DIR}/${qar_filename_exp} ]] ; then
       echo "QAR  ${QAR_RES_DIR}/${qar_filename_exp} exists"
       if [[ ${delete_data} == "true" ]] ; then
           echo "deleting ${DATA_DIR}/${dataset_type}/${dataset}/${variable} data"
           rm ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*.grib
           rm ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/*.nc
         else
            echo "the following files can be removed: "
            echo $(ls ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/* )
            ls ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/* > ${path_to_export}/files-to-be-removed-${dataset}-${variable}.txt
         fi
     fi
  else
    echo "ERROR: Unknow dataset type"
    exit 1
  fi
done


