#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Jesus Pena Izquierdo (jesus.pena@bsc.es)
#       Joan Sala Calero (joan.sala@bsc.es)

import configparser, glob
import os, sys, logging, logging.handlers
import pandas as pd
import numpy as np

from dqc_wrapper.scripts.rename_c3s_files import *
from dqc_wrapper.utils.utils_stats import multiparametric_valid_range, get_plausible_ranges
from dqc_wrapper.utils.utils_bokeh import BokehPlot
from dqc_wrapper.utils.utils_forms import get_metadata
from dqc_wrapper.utils.utils_files import save_result

regions = [ 'total', 'N_polar', 'N_midlat', 'N_hadley', 'S_hadley', 'S_midlat', 'S_polar' ]
conf_dir = '/data/joan/c3s512-wp1-datachecker/dqc_wrapper/QAR_dqc/conf'
forms_dir = '/data/cds-forms-c3s'
res_dir = '/shared/QAR_dqc/results'
logger = None

with open('/tmp/valid_ranges_stats.csv', 'w') as fo:
    for conf_file in sorted(glob.glob('{}/*.ini'.format(conf_dir))):

        # Read config
        try:
            c = configparser.RawConfigParser()
            c.read(conf_file)
            c_out = configparser.RawConfigParser()
            c_out.read(os.path.join(res_dir, os.path.basename(conf_file)))
            title = os.path.basename(conf_file).replace('.ini', '')
            cds_dataset = c['dataset']['cds_dataset']
            cds_variable = c['dataset']['cds_variable']
            cds_units = c['dataset']['cds_units']
            metadata = get_metadata(forms_dir, cds_dataset, cds_variable)
        except:
            print('ERROR: generating plot for {}'.format(conf_file))
            continue

        # Read csvs
        csv_hist = os.path.join(res_dir, os.path.basename(conf_file).replace('.ini', '-hist.csv'))
        csv_stats = csv_hist.replace('hist', 'stats')
        if not(os.path.exists(csv_hist) and os.path.exists(csv_stats)):
            continue
        df_stats = pd.read_csv(csv_stats)

        # Estimated valid_min/valid_max (conservative approach) - Catch exception
        valid_min, valid_max = None, None
        try:
            valid_min = float(c['valid_ranges']['valid_min'])
            valid_max = float(c['valid_ranges']['valid_max'])
        except:
            pass

        if not(valid_min is None) or not(valid_max is None):
            html = os.path.join(res_dir+'/ok', os.path.basename(conf_file).replace('.ini', '-valid_ranges.html'))
            if os.path.exists(html):
                os.system('cp {} /shared/QAR_dqc/results/ok_tol/'.format(html))

        control_limits = {}
        for reg in regions:
            try:
                mvr = multiparametric_valid_range(csv_path=res_dir, logarithmic_hist=[True],
                                                  dr_gap=np.arange(0.01, 0.3, 0.01), r_zero=np.arange(0.01, 0.1, 0.01),
                                                  k_gap=np.arange(1, 6, 2), extra=False, domain=reg)
                (vmin_nmax, vmax_nmax), (o_vmin, o_vmax), err, others = mvr.get(file=csv_hist, logger=logger)
                control_limits[reg] = {
                    'vmin_nmax': vmin_nmax,
                    'vmax_nmax': vmax_nmax,
                    'o_vmin': o_vmin,
                    'o_vmax': o_vmax,
                }
            except:
                control_limits[reg] = {
                    'vmin_nmax': None,
                    'vmax_nmax': None,
                    'o_vmin': None,
                    'o_vmax': None,
                }

        # Evaluate and return message
        res, msg = get_plausible_ranges(df_stats, control_limits['total']['vmin_nmax'], control_limits['total']['vmax_nmax'],
                                        valid_min, valid_max, cds_units, logger)
        outdir = os.path.join(res_dir, res)

        # Update ini file
        try:
            c_out.set('valid_ranges_result', 'res', res)
            c_out.set('valid_ranges_result', 'msg', msg)
            save_result(c_out, res_dir, os.path.basename(conf_file))
        except:
            print('Could not update ini file')

        # Final plots
        output_plot_html = os.path.join(outdir, os.path.basename(csv_hist).replace('-hist.csv', '-valid_ranges.html'))
        bp = BokehPlot(title, output_plot_html, csv_stats, csv_hist, cds_variable, cds_dataset,
                       valid_min, valid_max, control_limits, metadata, cds_units, msg, res)
        bp.plot_histograms()

        fo.write('{},{},{},{},{},{},{},{}\n'.format(os.path.basename(csv_hist).replace('-hist.csv', ''), valid_min, valid_max, o_vmin, o_vmax, err, res, msg))