#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#     
#     Authors: 
#       Joan Sala Calero (joan.sala@bsc.es)
#       Daniel Trujillo Viedma (daniel.trujillo@bsc.es)
#		Alejandro Alfonso Hernandez  (alejandro.alfonso@bsc.es)

import os
from textwrap import wrap
import configparser
from io import StringIO
from datetime import date, datetime

from reportlab.pdfbase.ttfonts import TTFont
from reportlab.pdfgen import canvas
from reportlab.lib.units import inch, cm
from reportlab.pdfbase.pdfmetrics import registerFont

from dqc_wrapper.utils.utils_files import rename_from_ini

# Header and footer (official C3S)
imgdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'img')

# Register official C3S font
fontdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'font')
registerFont(TTFont('Calibri', os.path.join(fontdir, 'Calibri.ttf')))

# Register official C3S explanations per text
disclaimers = {
    'file_format': 'This test looks at the file format. The recommended formats are GRIB2 or NETCDF4',
    'standard_compliance':'This test verifies whether the data comply with the metadata standard identified. It checks the definitions of the different metadata keys stored in the files and outputs the result in binary form. The summary of the result is given below, the log details are available in the following pages.',
    'completeness': 'This test verifies whether the data are complete in space (no unexpected missing values) and in time (all expected time steps are present). In case of an ocean variable (e.g. sea surface temperature), values are expected to be missing over land, thus a land/sea mask is used to determine missing values (data should be there) and unnecessary values (data is there but it should not be). Each dataset has a different land/sea mask. When the land/sea mask is not available, the test flags missing values also over land. Vice versa for land variables. In case of time completeness this data check looks for discrepancies in timestamps between the CDS Catalogue and the files delivered by the CDS-API download service. For seasonal forecasts datasets, the test handles the lead time structure. The output of the checks is given in binary form, summarized below, with log details available in the following pages.',
    'spatial_consistency': 'This test verifies whether there are discrepancies in the grid definition (step and grid type) between the metadata and the data of the variable files. "dual_resolution = yes" is used to indicate that the ERA5 variables have 2 spatial resolutions (reanalysis and uncertainty estimate) for the same variable, and that spatial consistency will be checked for both. For the other datasets, dual_resolution is set to "no" and therefore only one resolution is checked. For UERRA datasets the grid interval is expressed in meters. For the others, it is expressed in degrees. The output of the checks is given in binary form, summarized below, with log details available in the following pages.',
    'temporal_consistency': 'This test verifies whether there are discrepancies in the time dimension definition between the metadata and the data of the variable files. To do so, the checker software determines whether the correlative timestamps are consistent. The output of the checks is given in binary form, summarized below, with log details available in the following pages.',
    'valid_ranges': 'This test computes and displays the distribution of the data to detect any obvious value issues (e.g. data corruption, outliers) and illustrates their statistical properties. Any clear outlier can be identified by visually inspecting the histograms of the data distribution below. The histograms are available for the complete time series and for 6 different latitudinal bands (90 to 60, 60 to 30, 30 to 0, north and south). Below the histograms, an interactive boxplot, based on statistical definitions,is available. It gives an overview on the variability of the different statistical properties. The ranges of the physical plausibility are summarized below, with plots available in the following pages. The summary flags values that are outside defined boundaries. The boundaries are named "control limits" and "tolerance limits". The "control limits" are based on statistical analyses and depend on the distribution of the variable considered. The methodology is defined in appendix. The "tolerance limits" are completely independent of the distribution of the variable considered and are predefined acceptable boundaries based on physical considerations. WARNING: values very slightly outside the tolerance limits may appear because of data packing issues.'
}

"""
Convert a txt to a pdf file (reporting)
"""
def line_breaks(text, chars_per_line=100):
    ls = ''
    for t in text.split('\n'):
        pieces = wrap(t, chars_per_line, drop_whitespace=False)
        for p in pieces:
            ls = ls + p + '\n'
    return ls

"""
Convert a txt to a pdf file (reporting)
txt1_file: summary
txt2_file: detailed log file
"""
def save_txt_report(txt1_file, txt2_file, pdf_file, fstTitle='Summary of data checks'):

    # PDF report name
    print('INFO: Saving report to {}'.format(pdf_file))

    # Read content (detailed txt file messages)
    with open(txt1_file, 'r') as f:
        txt_str1 = f.read()
    with open(txt2_file, 'r') as f:
        txt_str2 = f.read()

    # Line breaks
    txt_str1 = line_breaks(txt_str1, 100)
    txt_str2 = line_breaks(txt_str2, 100)

    # Write PDF
    try:
        # Init
        c = canvas.Canvas(pdf_file)
        break_into_pages(txt_str1, c, 39, fstTitle, False, 15)
        break_into_pages(txt_str2, c, 39, 'Detailed log output', True, 15)

        # Save
        c.save()
    except:
        raise Exception('Could not convert txt to pdf. Check directory permissions')

"""
Iterate through results and export PDF files
"""
def save_all_pdf_reports(c, res_file):

    completeness_done = False
    files2zip = []

    for s in c.config.sections():
        if not('result' in s) and s!='general' and s!='dataset':
            # Spatial and temporal completeness go together for C3S
            if 'completeness' in s:
                if not(completeness_done):
                    files2zip.append(save_pdf_report('completeness', res_file, c.log_file, suffix='spatial_temporal_completeness'))
                    completeness_done = True

            # The other reports go alone
            else:
                files2zip.append(save_pdf_report(s, res_file, c.log_file))

    return files2zip

"""
Convert a ini to a pdf file (reporting)
res_file: result file in INI format (DQC output)
log_file: log file in TXT format (DQC output)
"""
def save_pdf_report(testname, res_file, log_file, suffix=''):

    # Rename pdf filename according to naming convention
    pdf_file = rename_from_ini(res_file, testname, 'pdf')

    if suffix != '':
        pdf_file = pdf_file.replace('completeness.pdf', suffix+'.pdf')
    print('INFO: Saving report to {}'.format(pdf_file))

    # Read+Filter content (result/config)
    res_str = filter_config(res_file, testname)

    # Try to get disclaimer (contains)
    dis_str = ''
    for k,v in disclaimers.items():
        if k in testname:
            dis_str = v

    # Read content (detailed log file messages)
    if not(log_file is None):
        log_str = ''
        with open(log_file, 'r') as f2:
            log_str = f2.read()
            log_str = filter_page(testname, log_str)

    # Write PDF
    c = canvas.Canvas(pdf_file)

    # Write 2 sections
    break_into_pages(line_breaks(" \n {} \n {} \n \n {}".format(dis_str, "_"*85, res_str)), c, 39, 'Summary of data checks', False, 15)
    if not(log_file is None):
        break_into_pages(line_breaks(log_str), c, 39, 'Detailed log output', True, 15)

    # Save
    c.save()

    return pdf_file

"""
Filter a config object
"""
def filter_config(res_file, testname):

    title = os.path.basename(res_file).replace('.ini', '')

    # Configuration object
    cf = configparser.RawConfigParser()
    cf.read(res_file)

    for s in cf.sections():
        # Remove log (unnecessary for CMS)
        if 'log' in cf[s]:
            cf[s].pop('log')

        # Remove sections of completeness without useful info for the user
        if 'completeness' in testname:
            if s == 'spatial_completeness' or s == 'temporal_completeness':
                cf.remove_section(s)

        if 'spatial_consistency' in testname and s == 'spatial_consistency':

            # Copy correct interval to item 'grid_interval'
            if cf['spatial_consistency_result']['res'] == 'ok':
                msg = cf['spatial_consistency_result']['msg']
                msg = msg.rsplit('=')[-1][:-1]

            else:
                msg = cf['spatial_consistency']['grid_interval']

            if not ',' in msg:
                msg = msg.replace('[', '').replace(']', '')

            if not 'meters' in msg or not 'degrees' in msg:
                if 'uerra' in title:
                    msg += ' meters'
                else:
                    msg += ' degrees'

            cf['spatial_consistency']['grid_interval'] = msg

            # Remove useless items in spatial_consistency section
            if cf[s].get('grid_mapping_name') is not None:
                cf[s].pop('grid_mapping_name')
            if 'era5' in title and not('era5-land' in title):
                pass
            else:
                if cf[s].get('dual_resolution') is not None:
                    cf[s].pop('dual_resolution')

        # Remove section (if not relevant for testXX)
        if not(testname in s):
            cf.remove_section(s)

    # Metadata
    time = 'generated on {}'.format(date.today())

    cf.add_section('metadata')
    cf.set('metadata', 'variable', title)
    cf.set('metadata', 'test', testname)
    cf.set('metadata', 'time', time)

    # Write config to string
    res_str = StringIO()
    cf.write(res_str)

    return res_str.getvalue().replace('\n', ' \n ')

"""
Filter page (by test name)
"""
def filter_page(s, page_str):
    out_str = ''
    s1 = s.replace('-', '_')
    s2 = s.replace('_', '-')

    for l in page_str.splitlines():
        iden = l.split(':')[0]

        if 'not implemented yet' in l:
            continue

        if s1 in iden or s2 in iden or s in iden:
            out_str += (l + '\n')
    return out_str

"""
Init PDF page (body)
"""
def init_page(c, leading, font_size=7):

    # Normal text prepare
    t = c.beginText()
    t.setFillColorRGB(0, 0, 0)
    t.setFont('Calibri', font_size)
    t.setCharSpace(0.5)
    t.setTextOrigin(40, 700)
    t.setLeading(leading)
    return t

"""
Init PDF page (title)
"""
def init_title(c, font_size=9):

    # TITLE text prepare - color-c3s: #941333
    t = c.beginText()
    t.setFillColorRGB(0.58, 0.075, 0.2)
    t.setFont('Calibri', font_size)
    t.setCharSpace(0.5)
    t.setTextOrigin(40, 725)
    return t

"""
Write PDF page
"""
def write_page(c, body, title, title_str, leading):

    # Header
    c.drawInlineImage(os.path.join(imgdir, "header.png"), 0, 755, height=1.2*inch, width=8.5*inch)

    # Text (title + body)
    title.textLine(title_str)
    c.drawText(title)
    c.drawText(body)

    # Footer
    c.drawImage(os.path.join(imgdir, "footer.png"), 0, 0, mask='auto', height=1.5*inch, width=8.5*inch)

    # Finalize
    c.showPage()

    return init_title(c), init_page(c, leading)

"""
Get page title/section
"""
def get_page_title(log_page, tit, page_count):

    if log_page:
        return tit + ' (page {})'.format(page_count)
    else:
        return tit

"""
Write a string into pages
"""
def break_into_pages(txtstr, c, lines_per_page, tit, log_page, leading, limit_pages=50):

    title = init_title(c, 16)
    body = init_page(c, leading, 11)
    count_lines = 0
    page_count = 1

    for l in txtstr.split('\n'):
        body.textLine(l)
        if count_lines == lines_per_page:
            # This is getting messy. Why does write_page initialize the title and
            # body objects with the default settings?
            title, body = write_page(c, body, title, get_page_title(log_page, tit, page_count), leading)
            title = init_title(c, 16)
            body = init_page(c, leading, 11)
            count_lines = 0
            page_count += 1
            if page_count == limit_pages:
                break
        else:
            count_lines+=1

    # Last page
    if count_lines != 0:
        write_page(c, body, title, get_page_title(log_page, tit, page_count), leading)

