#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)

import configparser, glob, re
import pandas as pd
from dqc_wrapper.utils.utils_forms import get_metadata
from dqc_wrapper.utils.utils_grib import *

conf_dir = '/data/joan/c3s512-wp1-datachecker/dqc_wrapper/QAR_dqc/conf'
forms_dir = '/data/cds-forms-c3s'

# Regex for units
pattern = re.compile('[\W_]+')

# Dataframe
df = pd.DataFrame(columns=['dataset','variable','units_c3s','units_grib','eccodes_units','same_units'])
i = 0
for conf_file in sorted(glob.glob('{}/*.ini'.format(conf_dir))):

    try:
        # Read config
        c = configparser.ConfigParser()
        c.read(conf_file)
        cds_dataset = c['dataset']['cds_dataset']
        cds_variable = c['dataset']['cds_variable']
        cds_units = c['dataset']['cds_units']
        metadata = get_metadata(forms_dir, cds_dataset, cds_variable)

        # Pick a grib file and check units
        files = glob.glob(os.path.join(c['general']['input'], c['general']['fpattern']))
        if len(files):
            gc = GribFileCheck(files[0])
            grib_units = gc.get_grib_units(sample=True)
            eq = pattern.sub('', cds_units.lower()) == pattern.sub('', grib_units.lower()) == pattern.sub('', metadata['unit'].lower())
            df.loc[i] = [cds_dataset, cds_variable, cds_units, grib_units, metadata['unit'], eq]
            i+=1
    except:
        pass
        #print('ERROR: '+conf_file)

# Group by variable in order to compare
df.to_csv('/tmp/units_per_dataset.csv')
with open('/tmp/units_per_variable.csv', 'w') as fp:
    df2 = df.groupby('variable')
    for v, v_df in df2:
        unique_units = list(set(v_df.units_c3s.values))
        fp.write('{},{},{}\n'.format(v, len(unique_units)==1, unique_units))
        print()