#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)
#

import logging
import glob
import os
import configparser

from dqc_wrapper.utils.utils_forms import get_metadata
from dqc_wrapper.utils.utils_bokeh import BokehPlot
from dqc_wrapper.utils.utils_stats import *

'''
Script run in order to re-plot html files (example, if CSV's change)

'''
# Folder with results
res_folder = '/shared/QAR_dqc/results'
conf_folder = '/data/joan/c3s512-wp1-datachecker/dqc_wrapper/QAR_dqc/conf'
forms_dir = '/data/cds-forms-c3s'
logger = logging.getLogger('redoplots')

for cf_file in glob.glob(os.path.join(conf_folder, '*.ini')):

    # Read config params
    config = configparser.ConfigParser()
    config.read(cf_file)
    cds_dataset = config['dataset']['cds_dataset']
    cds_variable = config['dataset']['cds_variable']

    # WMO ranges
    try:
        valid_min = float(config['valid_ranges']['valid_min'])
    except:
        valid_min = None
    try:
        valid_max = float(config['valid_ranges']['valid_max'])
    except:
        valid_max = None

    csv_hist = os.path.join(res_folder, os.path.basename(cf_file).replace('.ini', '-hist.csv'))
    csv_stats = csv_hist.replace('-hist.csv', '-stats.csv')
    out_html = csv_hist.replace('-hist.csv', '-valid_ranges.html')
    metadata = get_metadata(forms_dir, cds_dataset, cds_variable)

    # Re-do individual plots
    if os.path.exists(csv_hist) and os.path.exists(csv_stats):
        print(out_html)
        df_stats = pd.read_csv(csv_stats)
        df_hist = pd.read_csv(csv_hist)

        # Estimated valid_min/valid_max (conservative approach) - Catch exception
        try:
            estimated_valid_min, estimated_valid_max = exhaustive_vmin_vmax(csv_hist, perc=[25, 75])
            est_val_min_high = max(estimated_valid_min)
            est_val_min_low = min(estimated_valid_min)
            est_val_max_high = max(estimated_valid_max)
            est_val_max_low = min(estimated_valid_max)
        except Exception as e:
            print(str(e))

        # Defaults
        res = ''
        msg_data = 'Data range: [from {} to  {}]{}'.format(
            format_float(min(df_stats['Min'])), format_float(max(df_stats['Max'])), metadata['unit'])

        # Unknown ranges for the variable
        if valid_min is None and valid_max is None:
            if estimated_valid_min is None and estimated_valid_max is None:
                res = 'err'
                msg = 'Physical plausible range unknown. Unable to estimate the physical plausible range.'
            else:
                msg_est_vr = 'Estimated physical plausible range [from {} to  {}]{}'.format(
                    format_float(est_val_min_low), format_float(est_val_max_high), metadata['unit']
                )
                res = check_vmin_vmax(df_stats, est_val_min_low, est_val_max_high, logger,
                                      res_val='warn')
                msg = '{}. {}. '.format(msg_data, msg_est_vr)

                # Conclusion plausible range
                if res == 'ok':
                    msg += 'No potential outliers found.'
                else:
                    msg += 'Estimation found potential outliers.'

        # Valid range estimation (only if defined, well-known variable limits)               
        else:
            msg_vr = 'Known physical plausible range [from {} to  {}]{}'.format(
                format_float(valid_min), format_float(valid_max), metadata['unit'])
            res = check_vmin_vmax(df_stats, valid_min, valid_max, logger, res_val='err')
            msg = '{}. {}. '.format(msg_data, msg_vr)

            # Conclusion plausible range
            if res == 'ok':
                msg += 'No outliers found.'
            else:
                msg += 'Potential outliers found. Check log for details'

        # Final plots
        bp = BokehPlot(out_html, csv_stats, csv_hist, cds_variable, cds_dataset,
                       valid_min, valid_max, est_val_min_low, est_val_max_high, est_val_min_high, est_val_max_low,
                       metadata)
        bp.plot_histograms()
