source ~/.bashrc

# for each variable located in  /shared/cds_downloads/${dataset_type}/${dataset}
# 1- build the zip name that should be in the results folder
# for instance seasonal-monthly-single-levels/10m_u_component_of_wind would be
# seasonal-monthly-single-levels-10m_u_component_of_wind

# 2 - look in the QAR folder (/shared/QAR_dqc/results/) if there any matching zip file and for each of these
# remove the dataset-varname from the name itself and also the extension so we get the sistem for which the QAR exists
# then inform or remove the file in the /shared/cds_downloads/ for such system

source ~/.bashrc

# Filter
if [[ -z "$1" ]] && [[ -z "$2" ]] ; then
    echo "No filter arguments supplied"
    exit 1
fi

dataset_type=$1
dataset=$2
delete_data_files

DATA_DIR="/shared/cds_downloads"
QAR_RES_DIR="/shared/QAR_dqc/results"

echo "checking dir "${DATA_DIR}/${dataset_type}/${dataset}

for variable in `ls -1 ${DATA_DIR}/${dataset_type}/${dataset}*`
do
   echo "processing:"  $(basename ${variable})

   qar_filename_exp=${dataset}-$(basename ${variable})*-*.zip
   echo ${qar_filename_exp}
   for qar in `ls -1 ${QAR_RES_DIR}/${qar_filename_exp}`
   do
       qar_filename=$(basename ${qar})
       prefix=${dataset}-${variable}-
       datafile=${qar_filename#$( echo ${prefix})}
       datafile=${datafile%.zip}
       echo "QAR  ${qar_filename} exists"
       echo "the following files can be removed: " ${DATA_DIR}/${dataset_type}/${dataset}/${variable}/ "for system ${datafile}"
   done

done


