#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)
#

import configparser, glob
import os, sys, logging, logging.handlers
import pandas as pd
import numpy as np

from dqc_wrapper.scripts.rename_c3s_files import *
from dqc_wrapper.utils.utils_files import format_float
from dqc_wrapper.utils.utils_stats import multiparametric_valid_range, check_vmin_vmax
from dqc_wrapper.utils.utils_bokeh import BokehPlot
from dqc_wrapper.utils.utils_forms import get_metadata

conf_dir = '/data/joan/c3s512-wp1-datachecker/dqc_wrapper/QAR_dqc/conf'
forms_dir = '/data/cds-forms-c3s'
res_dir = '/shared/QAR_dqc/results'
logger = logging.getLogger()

with open('/tmp/valid_ranges_stats.csv', 'w') as fo:
    for conf_file in sorted(glob.glob('{}/*.ini'.format(conf_dir))):
        # Read config
        c = configparser.ConfigParser()
        c.read(conf_file)
        cds_dataset = c['dataset']['cds_dataset']
        cds_variable = c['dataset']['cds_variable']
        metadata = get_metadata(forms_dir, cds_dataset, cds_variable)

        # Read csvs
        csv_hist = os.path.join(res_dir, os.path.basename(conf_file).replace('.ini', '-hist.csv'))
        csv_stats = csv_hist.replace('hist', 'stats')
        if not(os.path.exists(csv_hist) and os.path.exists(csv_stats)):
            continue
        df_stats = pd.read_csv(csv_stats)

        # Estimated valid_min/valid_max (conservative approach) - Catch exception
        valid_min, valid_max = None, None
        try:
            valid_min = float(c['valid_ranges']['valid_min'])
            valid_max = float(c['valid_ranges']['valid_max'])
        except:
            pass

        vmin_nmax, vmax_nmax, o_vmin, o_vmax, others = None, None, None, None, None
        try:
            mvr = multiparametric_valid_range(csv_path=res_dir, logarithmic_hist=[True],
                                              dr_gap=np.arange(0.01, 0.3, 0.01), r_zero=np.arange(0.01, 0.1, 0.01),
                                              k_gap=np.arange(1, 6, 2), extra=False)
            (vmin_nmax, vmax_nmax), (o_vmin, o_vmax), err, others = mvr.get(file=csv_hist, logger=logger)
        except Exception as e:
            print(str(e))

        # Defaults
        res = ''
        msg_data = 'Data range: [from {} to  {}]{}'.format(
            format_float(min(df_stats['Min'])), format_float(max(df_stats['Max'])), metadata['unit'])

        # Unknown ranges for the variable
        if valid_min is None and valid_max is None:
            if vmin_nmax is None and vmax_nmax is None:
                res = 'err'
                msg = 'Physical plausible range unknown. Unable to estimate the physical plausible range.'
            else:
                res = check_vmin_vmax(df_stats, vmin_nmax, vmax_nmax, logger, res_val='warn')
                msg_est_vr = 'Estimated physical plausible range [from {} to {}]{}'.format(
                    format_float(vmin_nmax), format_float(vmax_nmax), metadata['unit']
                )
                msg = '{}. {}. '.format(msg_data, msg_est_vr)

                # Conclusion plausible range
                if res == 'ok':
                    msg += 'No potential outliers found.'
                else:
                    msg += 'Estimation found potential outliers.'

        # Valid range estimation (only if defined, well-known variable limits)
        else:
            res = check_vmin_vmax(df_stats, valid_min, valid_max, logger, res_val='err')
            msg_vr = 'Known physical plausible range [from {} to  {}]{}'.format(
                format_float(valid_min), format_float(valid_max), metadata['unit'])
            msg = '{}. {}. '.format(msg_data, msg_vr)

            # Conclusion plausible range
            if res == 'ok':
                msg += 'No outliers found.'
            else:
                msg += 'Potential outliers found. Check log for details'

        # Final plots
        output_plot_html = csv_hist.replace('-hist.csv', '-valid_ranges.html')
        bp = BokehPlot(output_plot_html, csv_stats, csv_hist, cds_variable, cds_dataset,
                       valid_min, valid_max, vmin_nmax, vmax_nmax, o_vmin, o_vmax,
                       metadata, msg, res)
        bp.plot_histograms()

        # Final message
        print(msg)

        fo.write('{},{},{},{},{},{},{},{}\n'.format(os.path.basename(csv_hist).replace('-hist.csv', ''), valid_min, valid_max, o_vmin, o_vmax, err, res, msg))