#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Jesus Pena Izquierdo (jesus.pena@bsc.es)
#       Joan Sala Calero (joan.sala@bsc.es)

import configparser, glob
import os, sys, logging, logging.handlers
import pandas as pd
import numpy as np

from dqc_wrapper.scripts.rename_c3s_files import *
from dqc_wrapper.utils.utils_stats import multiparametric_valid_range, get_plausible_ranges
from dqc_wrapper.utils.utils_bokeh import BokehPlot
from dqc_wrapper.utils.utils_forms import get_metadata

conf_dir = '/data/joan/c3s512-wp1-datachecker/dqc_wrapper/QAR_dqc/conf'
forms_dir = '/data/cds-forms-c3s'
res_dir = '/shared/QAR_dqc/results'
logger = None

with open('/tmp/valid_ranges_stats.csv', 'w') as fo:
    for conf_file in sorted(glob.glob('{}/*.ini'.format(conf_dir))):
        # Read config
        c = configparser.ConfigParser()
        c.read(conf_file)
        cds_dataset = c['dataset']['cds_dataset']
        cds_variable = c['dataset']['cds_variable']
        try:
            cds_units = c['dataset']['cds_units']
        except:
            cds_units = ""
        metadata = get_metadata(forms_dir, cds_dataset, cds_variable)

        # Read csvs
        csv_hist = os.path.join(res_dir, os.path.basename(conf_file).replace('.ini', '-hist.csv'))
        csv_stats = csv_hist.replace('hist', 'stats')
        if not(os.path.exists(csv_hist) and os.path.exists(csv_stats)):
            continue
        df_stats = pd.read_csv(csv_stats)

        # Estimated valid_min/valid_max (conservative approach) - Catch exception
        valid_min, valid_max = None, None
        try:
            valid_min = float(c['valid_ranges']['valid_min'])
            valid_max = float(c['valid_ranges']['valid_max'])
        except:
            pass

        vmin_nmax, vmax_nmax, o_vmin, o_vmax, others = None, None, None, None, None
        try:
            mvr = multiparametric_valid_range(csv_path=res_dir, logarithmic_hist=[True],
                                              dr_gap=np.arange(0.01, 0.3, 0.01), r_zero=np.arange(0.01, 0.1, 0.01),
                                              k_gap=np.arange(1, 6, 2), extra=False)
            (vmin_nmax, vmax_nmax), (o_vmin, o_vmax), err, others = mvr.get(file=csv_hist, logger=logger)
        except Exception as e:
            print(str(e))

        # Evaluate and return message
        res, msg = get_plausible_ranges(df_stats, vmin_nmax, vmax_nmax, valid_min, valid_max, cds_units, logger)

        # Final plots
        output_plot_html = csv_hist.replace('-hist.csv', '-valid_ranges.html')
        bp = BokehPlot(output_plot_html, csv_stats, csv_hist, cds_variable, cds_dataset,
                       valid_min, valid_max, vmin_nmax, vmax_nmax, o_vmin, o_vmax,
                       metadata, cds_units, msg, res)
        bp.plot_histograms()

        # Final message
        print(msg)

        fo.write('{},{},{},{},{},{},{},{}\n'.format(os.path.basename(csv_hist).replace('-hist.csv', ''), valid_min, valid_max, o_vmin, o_vmax, err, res, msg))
