#!/bin/bash

# Creation: 6/2016
# Authors: Nicola Cortesi and Raul Marcos
# Aim: to do a quality control of the dimension variables inside all NetCDF data file in a directory. Notice that this script is not able to find if a file is corrupt.
# I/O: you only have to specify where are the files you want to check and the correct names and size of the dimensions inside.
#      the output is a text file inside the directory where this script is run, whose name starts with "check_", with inside the results of the quality control.

var=psl #sfcWind                 # name of the variable to check
dat=ECMWFS4                      # dataset name
freq=6hourly                     # time step
#path=/esnas/exp/ECMWF/seasonal/0001/s004/m001/6hourly/sfcWind/    # its path
path=/esnas/exp/ecmwf/system4_m1/6hourly/psl/
#/esnas/exp/Meteofrance/seasonal/0001/s004/m001/6hourly/sfcWind Meteofrance S4
#/esnas/exp/ECMWF/seasonal/0001/s004/m001/6hourly/sfcWind ECMWF S4

suffix=01                        # this suffix is usually present in all S4 netCDF file, right begore the file extension
ext=.nc                          # file extension

yearStart=1981                   # time period to check inside the files
yearEnd=2015

nameLatitude=latitude            # name of the latitude dimension to check in the file
nLatitude=181 #128 #181 #256     # number of latitude values to check

nameLongitude=longitude          # name of the longitude dimension to check in the file
nLongitude=360 #256 #360 #512    # number of longitude values to check

nameMembers=ensemble             # name of the dimension with the model members to check in the file
nMembers=15 #51 #15              # number of members to check in the file
nMemberss=51			 # if there is another number of ensemble members that it is ok (should be set the same as nMembers if not)

nameLeadtimes=time               # name of the dimension with the forecast time to check in the file
nLeadtimes=861 #216              # number of forecast times to check in the file (215 complete days * 4 (6hourly) + 1 (for the last midnight)

printf "\t\n" > ktemp
printf "Checked variable: \t $var\n" >> ktemp
printf "Path: \t $path\n" >> ktemp
printf "Extension: \t $ext\n" >> ktemp
printf "Checked period: \t $yearStart-$yearEnd\n" >> ktemp
printf "\t\n" >> ktemp
printf ">>>>>>>>>>>>>>>>>>>>>>>>  Expected elements  <<<<<<<<<<<<<<<<<<<<<<<<\n"  >> ktemp
printf "\t\n" >> ktemp
printf "Latitude denomination: \t $nameLatitude\n" >> ktemp
printf "Number of latitude values: \t $nLatitude\n" >> ktemp
printf "Longitude denomination: \t $nameLongitude\n" >> ktemp
printf "Number of longitude values:\t $nLongitude\n" >> ktemp
printf "Ensemble denomination: \t $nameMembers\n" >> ktemp
printf "Number of ensemble values: \t $nMembers\n" >> ktemp
if [ $nMembers != $nMemberss ]; then 
printf "Alternative number of ensemble values: \t $nMemberss\n" >> ktemp
fi
printf "Lead-time denonomination: \t $nameLeadtimes\n" >> ktemp
printf "Number of lead-times: \t $nLeadtimes\n" >> ktemp

printf "\n\n >>>>>>>>>>>>>>>>>>>>>>>>   Checking  <<<<<<<<<<<<<<<<<<<<<<<< \n\n" | tee -a ktemp

outputfile=./check_${dat}_${var}_${freq}.txt

# function that detect the size of a dimension variable in a netCDF file:
#
# usage: get_size myNetCDF.nc myDimensionName
#
function get_size {
   ncks -m ${1} | grep -E -i ": ${2}, size =" | cut -d ' ' -f 7 | uniq
}

# check file existence:
for ((i=$yearStart;i<=$yearEnd;i++))
do
for j in 01 02 03 04 05 06 07 08 09 10 11 12
do
  ok1=0; ok2=0; ok3=0; ok4=0 ; ok5=0

 # echo $path${var}_$i$j$suffix$ext > ktemp
  if [ ! -f $path${var}_$i$j$suffix$ext ]
    then
    echo ${var}_$i$j$suffix$ext ": >>>>> Missing file <<<<<" | tee -a ktemp
  else
      nMemb=`get_size $path${var}_$i$j$suffix$ext $nameMembers`
      nLead=`get_size $path${var}_$i$j$suffix$ext $nameLeadtimes`
      nLat=`get_size $path${var}_$i$j$suffix$ext $nameLatitude`
      nLon=`get_size $path${var}_$i$j$suffix$ext $nameLongitude`

      if [ -z $nLat ]; then # check if nLat is empty
        echo ${var}_$i$j$suffix$ext ": there is no dimension called " $nameLatitude " inside this file " | tee -a ktemp  
      elif [ ${nLat//[[:blank:]]/} -ne $nLatitude ]; then
        echo ${var}_$i$j$suffix$ext ": "$nameLatitude" dimension has " $nLat " values instead of " $nLatitude  | tee -a ktemp  
      else
        ok1=1
      fi

      if [ -z $nLon ]; then # check if nLon is empty
        echo ${var}_$i$j$suffix$ext ": there is no dimension called " $nameLongitude " inside this file " | tee -a ktemp 
      elif [ ${nLon//[[:blank:]]/} -ne $nLongitude ]; then
        echo ${var}_$i$j$suffix$ext ": "$nameLongitude "dimension has " $nLon " values instead of " $nLongitude | tee -a ktemp         
      else
        ok2=1
      fi

      if [ -z $nMemb ]; then # check if nMemb is empty
        echo ${var}_$i$j$suffix$ext ": there is no dimension called " $nameMembers " inside this file "  | tee -a ktemp
      elif [ ${nMemb//[[:blank:]]/} -ne $nMembers -a  ${nMemb//[[:blank:]]/} -ne "$nMemberss" ]; then
        echo ${var}_$i$j$suffix$ext ": "$nameMembers "dimension has " $nMemb " values instead of " $nMembers  | tee -a ktemp            
      else
        ok3=1
      fi

      if [ -z $nLead ]; then
        echo ${var}_$i$j$suffix$ext ": there is no dimension called " $nameLeadtimes " inside this file " | tee -a ktemp
      elif [ ${nLead//[[:blank:]]/} -ne $nLeadtimes ]; then
        echo ${var}_$i$j$suffix$ext ": "$nameLeadtimes " dimension has " $nLead " values instead of " $nLeadtimes | tee -a ktemp       
      else
        ok4=1
      fi
  

      if [ $ok1 -eq 1 ] && [ $ok2 -eq 1 ] && [ $ok3 -eq 1 ] && [ $ok4 -eq 1 ]; then
         echo ${var}_$i$j$suffix$ext ": ok" | tee -a ktemp
      fi
  fi
done
done

column -t -s $'\t' ktemp > $outputfile
rm ktemp

### other common changes to netCDF that can be useful:
# # convert a 6-hourly file to a daily one:
# for file in *; do cdo daymean $file /esnas/exp/ecmwf/system4_m1/daily_mean/sfcWind_f6h/$file

# cdo daymean <file_6-hourly.nc> <file_daily.nc>
# cdo showdate <file.nc>
# for file in *; do ncrename -d $file ; done

# convert a .nc file of ECMWF S4 after downloading:
#file=_grib2netcdf-atls01-95e2cf....nc ; ncrename -v .msl,psl -d .number,ensemble -v .number,realization $file ; date=$(cdo showdate $file | cut -f3 -d" " | sed -e s/-//g) ; mv $file /esnas/exp/ecmwf/system4_m1/6hourly/psl/psl_${date}.nc ; cdo daymean /esnas/exp/ecmwf/system4_m1/6hourly/psl/psl_${date}.nc /esnas/exp/ecmwf/system4_m1/daily_mean/psl_f6h/psl_${date}.nc ; cdo monmean /esnas/exp/ecmwf/system4_m1/daily_mean/psl_f6h/psl_${date}.nc /esnas/exp/ecmwf/system4_m1/monthly_mean/psl_f6h/psl_${date}.nc

#file=_grib2netcdf-atls01-95e2cf....nc
#ncrename -v .msl,psl -d .number,ensemble -v .number,realization $file
#date=$(cdo showdate $file | cut -f3 -d" " | sed -e s/-//g)
#mv $file /esnas/exp/ecmwf/system4_m1/6hourly/psl/psl_${date}.nc
#cdo daymean /esnas/exp/ecmwf/system4_m1/6hourly/psl/psl_${date}.nc /esnas/exp/ecmwf/system4_m1/daily_mean/psl_f6h/psl_${date}.nc
#cdo monmean /esnas/exp/ecmwf/system4_m1/daily_mean/psl_f6h/psl_${date}.nc /esnas/exp/ecmwf/system4_m1/monthly_mean/psl_f6h/psl_${date}.nc
