#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
# Barcerlona Supercomputing Center - Centro Nacional de Supercomputacion
#        Earth Sciences Department - Computational Earth Sciences
#     
#     Authors: Ivan Jose Cernicharo Ortiz (ivan.cernicharo@bsc.es)
#             -
#             -
#

import sys
import logging
import os
import subprocess

class spatial_completeness_test:

    def __init__(self,path,type=None):
        self.type = type
        self.path = path
        self.files = self.dir_or_file()

    def dir_or_file(self):
        if os.path.isfile(self.path):
            print("Checking file")
            print(self.path)
            # do the file checking
            return [self.path]

        elif os.path.isdir(self.path):
            print("Found the following files to check ")
            # list all files in the folder
            list_files = self.list_files_folder()
            print(list_files)
            return list_files

    def list_files_folder(self):
        list_files = []
        
        if os.path.exists(self.path) and os.path.isdir(self.path):
            for root, dirs, files in os.walk(self.path):
                    for f in files:
                        if f.endswith("grib") or f.endswith("nc"):
                            list_files.append(os.path.join(root,f))
            
            return sorted(list_files)
        else:
            raise Exception('\n\tGiven path is not a valid path\n')

    def missing_values_netcdf(self,ncitem,e):
        var = ncitem.variables[e]
        variable = var[:]
        missing = variable.missing_value
        if var.mask == False:
            # If the variable does not have a mask
            datamiss = len(variable==missing)
            mk = False if len(datamiss) == 0 else True
        else:
            # If the variable does have a mask
            data_masked = variable.mask
            data = variable.data
            real_data = data[~data_masked]
            datamiss = len(real_data==missing)
            mk = False if len(datamiss) == 0 else True

        return datamiss,mk

    def netcdf_test(self,args):
        import netCDF4
        import numpy as np

        output = list()


        for ncfile in args:
            ncitem = netCDF4.Dataset(f)
            test_var = [item for item in ncitem.variables.keys() \
                if item not in ncitem.dimensions.keys()]
            if test_var != []:
                for e in test_var:
                    datamiss,mk = self.missing_values_netcdf(ncitem,e)
                    if mk == True:
                        string = ncfile.split('/')[-1] + ' variable '
                        string += e + ' has missing data ' + datamiss
                        output.append('\n'.join([ncfile.split('/')[-1],string,e]))

        if len(output) == 0:
            stringA = '\nNO GAPS FOUND! HURRAY!'
            logging.info(stringA)

        else:
            stringA = 'n' + '\n\n'.join(output)
            logging.error(stringA)

    def missing_values_grib(self,msgs):
        # You should check how to see if a message has a mask

        output = list()
        for i,msg in enumerate(msgs):
            vals = msg['values']
            valmiss = msg['missingValue']
            numofmisscalc = len(vals[vals==valmiss])
            numofmiss = msg['numberOfMissing']

            if numofmisscalc == numofmiss and numofmiss > 0:
                string = 'Message number %s has %s num of missing values' %(i+1,numofmiss)
                output.append(string)
            
        if len(output) != 0:
            stringA = '\n' + '\n'.join(output)
            return stringA,'ER'            

        else:
            stringA = '\nNo missing values'
            return stringA,'OK'

    def grib_test(self,args):
        import utils.grib_content as gcont
        
        oute = list()
        for item in args:
            gfile = gcont.grib_content(item)
            msgs = gfile.return_all_msg()

            string,wrn = self.missing_values_grib(msgs)

            if wrn == 'ER':
                stringA = '\n' + ' - '*20
                stringA += '\nFILE %s has missing values. More precisely\n\n' 
                stringA += string

            try:
                oute.append(stringA)
            except:
                stringA = oute

        if len(oute) == 0:
            stringB = '\nNO GAPS FOUND! '
            logging.info(stringA)

        else:
            stringB = 'n' + '\n\n'.join(oute)
            print("Some errors were found")
            print(stringB)
            logging.error(stringA)


    def initiate_logging(self):
        # Initiate the logging
#        time0 = self.timefortitle
        path_logs = os.path.join(
            '/data',
            'DQC_LOGS'
        )

        if not os.path.exists(path_logs):
            os.makedirs(path_logs)

        default_log_name = '_'.join(
            [
                'LOG_TEST02_SPATIAL_COMPLETENESS_CHECK',
#                (self.type).upper(),
#                time0,
                '.log'
            ]
        )

        default_path = os.path.join(
            path_logs,
            default_log_name
        )
        
        logging.basicConfig(
            filename=default_path,
            format='%(levelname)s:%(asctime)s:%(message)s', 
            level=logging.INFO
        )

        stringA = '\n' + '#'*80 + '\n' + '#'*80
        stringA += '\n RUNNING SPATIAL COMPLETENESS TEST FOR FOLDER:'
        stringA += '\n ' + self.path
        stringA += '\n AGAINST ' + (self.type).upper() + ' DQC '
#        stringA += '\n EXECUTION TIME: ' + self.timeforinfo
        stringA += '\n' + '#'*80

    def main(self):
        if self.type != None:                

            if self.type == 'netcdf':
                ncfiles = [item for item in self.files if item.endswith('.nc')]
                if ncfiles != []:
                    self.initiate_logging()
                    self.netcdf_test(ncfiles)
                else:
                    raise Exception('\n\tNo NetCDF files in the given input\n')

            if self.type == 'grib':
                gribfiles = [item for item in self.files if item.endswith('.grib')]
                if gribfiles != []:
                    self.initiate_logging()
                    self.grib_test(gribfiles)
                else:
                    raise Exception('\n\tNo GRIB files in the given input\n')

        else:
            raise Exception('\n\tStandard type has not been given by the user\n')

if __name__ == "__main__":
    sys.exit()
