#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)
#       Jesus Pena Izquierdo (jesus.pena@bsc.es)

import numpy as np
import pandas as pd

from dqc_wrapper.utils.utils_files import format_float

"""
Check for min/max per file (values are given)
"""
def check_vmin_vmax(df_stats, valid_min, valid_max, logger, res_val='err'):
    res = 'ok'
    for fname, row in df_stats.iterrows():
        # Check ranges (1 or 2)
        outlier = False
        if not(valid_min is None) and row['Min'] < valid_min:
            outlier = True
        if not(valid_max is None) and row['Max'] > valid_max:
            outlier = True

        if outlier:
            logger.error('{} contains outliers'.format(fname))
            logger.error('observed [{} to {}] vs valid [{} to {}]'.format(format_float(row['Min']), format_float(row['Max']),
                                                                        format_float(valid_min), format_float(valid_max)))
            res = res_val
    return res

"""
Producing a list of statistical estimators (on the data)
Stats needed for the boxplot
"""
def get_basic_stats(x, b):

    # Stats from histogram (more efficient, almost same result)
    q1, Med, q3 = quantiles_from_hist(b, x)
    IQR = q3 - q1

    return [Med, q1, q3, IQR]

"""
Get approximate statistical indicators from histogram
- This function computes the position of the quantiles given in the list quantiles.
- It linearly interpolates from the histogram to the corresponding values (i.e. no discretization issue)  
"""
def quantiles_from_hist(xm, nx, quantiles=[0.25, 0.5, 0.75]):

    dx = np.diff(xm)[0]  # xbin length
    N = np.sum(nx)  # total number of elements
    rx_cumsum = np.cumsum(nx) / N  # density distribution

    x_q = np.array([])
    for qi in quantiles:
        i_qi1 = np.argwhere(rx_cumsum > qi)[0].item()
        i_qi0 = i_qi1 - 1

        rx_qi0 = rx_cumsum[i_qi0]
        rx_qi1 = rx_cumsum[i_qi1]

        drx_qi01 = rx_qi1 - rx_qi0
        drx_qi = qi - rx_qi0
        k_drx_qi = drx_qi / drx_qi01

        x_qi = xm[i_qi0] + dx * (k_drx_qi + 1 / 2)
        x_q = np.append(x_q, x_qi)

    return x_q

"""
Try different combinations of (dr_gap, r_zero) to find gaps in distribution and better estimate vmin/vmax
"""
def exhaustive_vmin_vmax(csv_hist):

    # Exhaustive min/max find
    dr_gap = [0.5, 0.25, 0.1, 0.05, 0.025, 0.01, 0.005]
    r_zero = [0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001]
    valid_min = 999999999999
    valid_max = -999999999999

    for d in dr_gap:
        for r in r_zero:
            vr = valid_range(csv_hist, domain='total', dr_gap=d, r_zero=r, k_gap=1)
            try:
                mn, mx = vr.get()
                valid_max = max(valid_max, mx)
                valid_min = min(valid_min, mn)
            except:
                pass # method failed for given parameters (let's keep trying)

    if valid_min == 999999999999 or valid_max == -999999999999:
        return None, None
    else:
        return valid_min, valid_max

"""
Class used for estimating the valid ranges
Possible values for testing dr_gap and r_zero 
# dr_gap = [0.5 0.25 0.1 0.05 0.025 0.01 0.005]
# r_zero = [0.05 0.01 0.005 0.001 0.0005 0.0001]
"""
class valid_range:

    def __init__(self, csv_hist, domain='total', dr_gap=0.05, r_zero=0.01, k_gap=1):

        self.dr_gap = dr_gap  # Percentile distance as definition of outlier gap
        self.r_zero = r_zero  # Percentile definition of start/end of the data distribution

        # This depends on the number of elements, not use r_zero < 0.0001
        self.k_gap = k_gap  # Coefficient to multiply the estimated outlier distance
        self.csv_hist = csv_hist  # file name of csv file to process
        self.domain = domain  # domain of the csv file to be processed ['total', 'S_polar','S_midlat',...]

        # Read csv file
        data0 = pd.read_csv(csv_hist, index_col=0)
        data = data0.loc[self.domain]
        xedge = data.index.values.astype('float64')  # bin edges
        self.xm = xedge[:-1] + np.diff(xedge) / 2  # bin centre
        self.nx = data.values[:-1]  # bin counts

        self.N = np.sum(self.nx)  # total number of elements
        self.quantiles = np.cumsum(self.nx) / self.N  # density distribution. percentiles

        # Percentiles selected for the gap definition
        self.q_left = [self.r_zero, self.r_zero + self.dr_gap]
        self.q_right = [1 - self.dr_gap - self.r_zero, 1 - self.r_zero]

    """
    # This function computes the outlier gaps for the left and right tails of the distribution.
    # (r_gap => dx_left, dx_right)
    # This is the minimum size of data distribution gaps in each tail to consider data clusters separated
    # from the main distribution as real outliers.
    # This outlier_gap is defined as the distance between especified quantiles:
    # For the left outlier_gap: abs( x_q[r_zero] - x_q[r_zero+dr_gap] )
    # For the right outlier_gap: abs( x_q[1-dr_gap-r_zero] - [1-r_zero] )
    """
    def outlier_gaps(self):
        # Exact quantile positions for left and right bands
        xq_left = quantiles_from_hist(self.xm, self.nx, quantiles=self.q_left)
        xq_right = quantiles_from_hist(self.xm, self.nx, quantiles=self.q_right)

        # outlier gap for left band
        dx_left = self.k_gap * np.diff(xq_left).item()

        # outlier gap for right band
        dx_right = self.k_gap * np.diff(xq_right).item()

        return dx_left, dx_right, xq_left, xq_right

    """
    This function search for gaps greater than dx_left and dx_right at each side of the distribution.
    """
    def find_gaps(self):

        dx_left, dx_right, _, _ = self.outlier_gaps()

        # For left side:
        # --------------
        # Bins to the left of r_zero
        i_left = np.argwhere(self.quantiles < self.r_zero)[:, 0]
        nx_left = self.nx[i_left]
        if (nx_left == 0).sum() > 0:
            # Index of last bins to the left of r_zero with nx = 0
            i_zero = np.argwhere(nx_left == 0)[-1].item()

            # Bins to the left of i_zero
            i_left0 = i_left[:i_zero]
            nx_left0 = self.nx[i_left0]
            xm_left0 = self.xm[i_left0]

            # Searching for a group of bins with nx=0 (i.e. n_gap=0) and extension < dx_left
            n_gap = 1
            j = -1  # counting down from i_zero to the first elements. From right to left
            while n_gap:
                i0_gap_left = j  # -1
                xm0_gap_left = xm_left0[i0_gap_left]  # right edge of the gap
                xm1_gap_left = xm0_gap_left - dx_left  # left edge of the gap

                if xm1_gap_left >= xm_left0[0]:
                    i1_gap_left = np.argwhere(xm_left0 - xm1_gap_left < 0)[-1].item()  # index of the left edge element
                    n_gap = np.sum(nx_left0[i1_gap_left: i0_gap_left])  # total number of counts within the gap
                    j -= 1
                else:
                    raise Exception('Estimated valid left range out of histogram limit :o\nExtension of histogram left limit required')
                    break
        else:
            xm0_gap_right, xm1_gap_right = np.nan, np.nan
            raise Exception('Observed range goes beyond histogram left limit :o\nExtension of histogram left limit required')

        # For right side:
        # ----------------
        # Bins to the right of 1-r_zero
        i_right = np.argwhere(self.quantiles > 1 - self.r_zero)[:, 0]
        nx_right = self.nx[i_right]

        if (nx_right == 0).sum() > 0:
            # Index of first bins to the right of 1-r_zero with nx = 0
            i_zero = np.argwhere(nx_right == 0)[0].item()
            # Bins to the right of 1-i_zero
            i_right0 = i_right[i_zero:]
            nx_right0 = self.nx[i_right0]
            xm_right0 = self.xm[i_right0]

            # Searching for a group of bins with nx=0 (i.e. n_gap=0) and extension < dx_right
            n_gap = 1
            j = 0
            while n_gap:
                i0_gap_right = j  # 0
                xm0_gap_right = xm_right0[i0_gap_right]  # left edge of the gap
                xm1_gap_right = xm0_gap_right + dx_right  # right edge of the gap

                if xm1_gap_right <= xm_right0[-1]:
                    i1_gap_right = np.argwhere(xm_right0 - xm1_gap_right > 0)[
                        0].item()  # index of the right edge element
                    n_gap = np.sum(nx_right0[i0_gap_right: i1_gap_right])  # total number of counts within the gap
                    j += 1

                else:
                    raise Exception('Estimated valid right range out of histogram limit :o\nExtension of histogram right limit required')
                    break

        else:
            xm0_gap_right, xm1_gap_right = np.nan, np.nan
            raise Exception('Observed range goes beyond histogram right limit :o\nExtension of histogram right limit required')

        return xm1_gap_left, xm0_gap_left, xm0_gap_right, xm1_gap_right

    '''
    Main function
    '''
    def get(self):

        # Get outer boundaries of the empty gaps.
        xm1_gap_left, _, _, xm1_gap_right = self.find_gaps()

        return xm1_gap_left, xm1_gap_right
