#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
# Barcerlona Supercomputing Center - Centro Nacional de Supercomputacion
#        Earth Sciences Department - Computational Earth Sciences
#     
#     Authors: Ivan Jose Cernicharo Ortiz (ivan.cernicharo@bsc.es)
#             -
#             -
#

import sys
import logging
import os
import subprocess

def return_extension(file_in):
    dict_of_extensions = {
        'grib':'GRIB',
        'nc':'NetCDF',
        'zip':'Zipped'
    }
    
    ext = file_in.split('.')[-1]
    string = "%s extension (.%s)" %(dict_of_extensions[ext],ext)
    
    return string

class file_format_check:

    def __init__(self,path):
        self.path = path
        self.files = self.dir_or_file()
#        print("Files or directory checked:")
        self.gridded_files = ''
        self.other_files = ''
        self.formats1 = ''
        self.formats2 = ''

    def dir_or_file(self):
        if os.path.isfile(self.path):
            print("Checking file")
            print(self.path)
            # do the file checking
            return [self.path]

        elif os.path.isdir(self.path):
            print("Found the following files to check ")
            # list all files in the folder
            list_files = self.list_files_folder()
            print(list_files)
            return list_files

    def list_files_folder(self):
        list_files = []
        if os.path.exists(self.path):
            if os.path.isfile(self.path) and self.path.endswith("grib") or self.path.endswith("nc") or self.path.endswith("grb"):
                list_files.append(self.path) 
            if os.path.isdir(self.path) or os.path.isfile(self.path):
                for root, dirs, files in os.walk(self.path):
                    print(files)
                    for f in files:
                        if f.endswith("grib") or f.endswith("nc") or f.endswith("grb"):
                            list_files.append(os.path.join(root,f))
            
                return sorted(list_files)

            else:
                raise Exception('\n\tGiven path is not a valid path\n')

    def cdo_return_format(self):
        """This function uses the CDO module, with command sinfon to retrieve 
        file format
        """

        from cdo import Cdo

        CDO = Cdo()
        files_format = list()
        f = lambda x: 'File format' in x

        for item in self.gridded_files:
            file_format = list(filter(f,CDO.sinfon(input=item)))[0]
            out0 = file_format.replace('File format:','').strip()
            output = ": ".join([return_extension(item),out0])
            files_format.append(output)
        
            if any( "File format : GRIB" in s for s in files_format ) : 
                 files_format.append("Warning: grib1 being deprecated, the recommendation is to use grib2")
            if any( "File format : NetCDF2" in s for s in files_format ) : 
                 files_format = [ s.replace("NetCDF2","NetCDF3 64-bit offset") for s in files_format ]
        return set(files_format)

    def file_return_format(self):
        # This function uses the file function from bash to check format
        from subprocess import Popen
        
        files_format = list()

        for item in self.other_files:
            cmd0 = "file -b %s" %item

            p0 = Popen(
                cmd0, shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT
                )
            
            out0, out1 = p0.communicate()
            output = ": ".join([return_extension(item),str(out0)])

            if "\n" in output:
                output = output.replace("\n",'')

            if "Zipped extension" in output:
                import re

                cmd1 = "unzip -l %s" %item

                p1 = Popen(
                    cmd1, shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT
                    )

                out1, _ = p1.communicate()

                additional_ext = list()

                for item in list(re.findall('[.]\w+',str(out1))):

                    if not item.endswith('zip'):
                        additional_ext.append(item)

                additional_ext = sorted(additional_ext)
                additional_ext = "(" + ', '.join(additional_ext) + ")"

                output += " " + additional_ext
            
            files_format.append(output)


        return set(files_format)

    def merge_dictionaries(self):
        x,y = self.formats1,self.formats2
        z = x.copy()
        z.update(y)
        return z

    def initiate_logging(self):
        # Initiate the logging
#        time0 = self.timefortitle
        path_logs = os.path.join(
            '/data',
            'DQC_LOGS'
        )

        if not os.path.exists(path_logs):
            os.makedirs(path_logs)

        default_log_name = '_'.join(
            [
                'LOG_TEST01_STANDARD_CHECK',
#                (self.type).upper(),
#                time0,
                '.log'
            ]
        )

        default_path = os.path.join(
            path_logs,
            default_log_name
        )
        
        logging.basicConfig(
            filename=default_path,
            format='%(levelname)s:%(asctime)s:%(message)s', 
            level=logging.INFO
        )

        stringA = '\n' + '#'*80 + '\n' + '#'*80
        stringA += '\n RUNNING FILE FORMAT TEST FOR FOLDER:'
        stringA += '\n ' + self.path
#        stringA += '\n EXECUTION TIME: ' + self.timeforinfo
        stringA += '\n' + '#'*80

        logging.info(stringA)

    def return_extensions_format(self):
        try: 
            files = self.files
        except: 
            files = self.list_files_folder()
#        print(files)
        #unique_extensions = set(files)

        self.initiate_logging()

        files_dat1,files_dat2 = list(),list()
        for item in files:
            conditional = item.endswith('.grib') or item.endswith('.grb') or item.endswith('.nc')
            if conditional:
                files_dat1.append(item)
            else:
                files_dat2.append(item)
        
        if files_dat1 != []:
            self.gridded_files = files_dat1
            self.formats1 = self.cdo_return_format()
        
        if files_dat2 != []:
            self.other_files = files_dat2
            self.formats2 = self.file_return_format()

        if self.formats2 != '' and self.formats1 != '':
            unique_formats = self.merge_dictionaries()
        
        elif self.formats2 == '' and self.formats1 != '':
            unique_formats = self.formats1
        
        elif self.formats2 != '' and self.formats1 == '':
            unique_formats = self.formats2

        try: 
            unique_formats 
        except:
            print("No files found with grb, grib or nc extension, exiting")
            exit


        
#        string = '0: File format and extensions found:\n'
        string = '\n'
        unique_formats = sorted(unique_formats)
        for item in unique_formats:
            string += '\t%s\n' %item

        logging.info(string)

        return string

if __name__ == "__main__":
    sys.exit()
