#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#     
#     Authors: 
#       Joan Sala Calero (joan.sala@bsc.es)

import os
from textwrap import wrap
import configparser
from io import StringIO
from datetime import date

from reportlab.pdfbase.ttfonts import TTFont
from reportlab.pdfgen import canvas
from reportlab.lib.units import inch, cm
from reportlab.pdfbase.pdfmetrics import registerFont

# Header and footer (official C3S)
imgdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'img')

# Register official C3S font
fontdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'font')
registerFont(TTFont('Calibri', os.path.join(fontdir, 'Calibri.ttf')))

# Register official C3S explanations per text
disclaimers = {
	'file_format': 'This test looks at the file format. The recommended formats are GRIB2 or NETCDF4_CLASSIC',
	'standard_compliance':'This test verifies whether the data comply with the metadata standard identified. It checks the definitions of the different metadata keys stored in the files and outputs the result in binary form. The summary of the result is given below, the log details are available in the following pages',
	'completeness': 'This test verifies whether the data are complete in space (no unexpected missing values) and in time (all expected time steps are present). In case of an ocean variable (e.g. sea surface temperature), values are expected to be missing over land, thus a land/sea mask is used to determine missing values (data should be there) and unnecessary values (data is there but it should not be). Each dataset has a different land/sea mask. When the land/sea mask is not available, the test flags missing values also over land. Vice versa for land variables. In case of time completeness this data check looks for discrepancies in timestamps between the CDS Catalogue and the files delivered by the CDS-API download service. For seasonal forecasts datasets, the test handles the lead time structure. The output of the checks is given in binary form, summarized below, with log details available in the following pages.',
	'spatial_consistency': 'This test verifies whether there are discrepancies in the grid definition (step and grid type) between the metadata and the data of the variable files. The output of the checks is given in binary form, summarized below, with log details available in the following pages.',
	'temporal_consistency': 'This test verifies whether there are discrepancies in the time dimension definition between the metadata and the data of the variable files. To do so, the checker software determines whether the correlative timestamps are consistent. The output of the checks is given in binary form, summarized below, with log details available in the following pages.',
	'valid_ranges': 'This test computes and displays the distribution of the data to detect any obvious value issues (e.g. data corruption, outliers) and illustrates their statistical properties. Any clear outlier can be identified by visually inspecting the histograms of the data distribution below. The histograms are available for the complete time series and for 6 different latitudinal bands (90 to 60, 60 to 30, 30 to 0, north and south). Below the histograms, an interactive boxplot, based on statistical definitions,is available. It gives an overview on the variability of the different statistical properties. The ranges of the physical plausibility are summarized below, with plots available in the following pages. The summary flags values that are outside defined boundaries. The boundaries are named "control limits" and "tolerance limits". The "control limits" are based on statistical analyses and depend on the distribution of the variable considered. The methodology is defined in appendix. The "tolerance limits" are completely independent of the distribution of the variable considered and are predefined acceptable boundaries based on physical considerations. The WMO defines tolerance limits in [whoever remembers where the limits are defined could insert the reference?]'
}

"""
Convert a txt to a pdf file (reporting)
"""
def line_breaks(text, chars_per_line=150):
	ls = ''
	for t in text.split('\n'):
		pieces = wrap(t, chars_per_line, drop_whitespace=False)
		for p in pieces:
			ls = ls + p + '\n'
	return ls

"""
Convert a txt to a pdf file (reporting)
txt1_file: summary
txt2_file: detailed log file
"""
def save_txt_report(txt1_file, txt2_file, pdf_file):

	# PDF report name
	print('INFO: Saving report to {}'.format(pdf_file))

	# Read content (detailed txt file messages)
	with open(txt1_file, 'r') as f:
		txt_str1 = f.read()
	with open(txt2_file, 'r') as f:
		txt_str2 = f.read()

	# Line breaks
	txt_str1 = line_breaks(txt_str1)
	txt_str2 = line_breaks(txt_str2)

	# Write PDF
	try:
		# Init
		c = canvas.Canvas(pdf_file)
		break_into_pages(txt_str1, c, 58, 'Summary of data checks', False, 15)
		break_into_pages(txt_str2, c, 58, 'Detailed log output', True, 10)

		# Save
		c.save()
	except:
		raise Exception('Could not convert txt to pdf. Check directory permissions')

"""
Iterate through results and export PDF files
"""
def save_all_pdf_reports(c, res_file):

	completeness_done = False
	files2zip = []

	for s in c.config.sections():
		if not('result' in s):
			# Spatial and temporal completeness go together for C3S
			if 'completeness' in s:
				if not(completeness_done):
					files2zip.append(save_pdf_report('completeness', res_file, c.log_file, suffix='spatial_temporal_completeness'))
					completeness_done = True

			# The other reports go alone
			else:
				files2zip.append(save_pdf_report(s, res_file, c.log_file))

	return files2zip

"""
Convert a ini to a pdf file (reporting)
res_file: result file in INI format (DQC output)
log_file: log file in TXT format (DQC output)
"""
def save_pdf_report(testname, res_file, log_file, suffix=''):

	# PDF report name
	pdf_file = res_file.replace('.ini', '-{}.pdf'.format(testname))
	if suffix != '':
		pdf_file = pdf_file.replace('completeness.pdf', suffix+'.pdf')
	print('INFO: Saving report to {}'.format(pdf_file))

	# Read+Filter content (result/config)
	res_str = filter_config(res_file, testname)

	# Try to get disclaimer (contains)
	dis_str = ''
	for k,v in disclaimers.items():
		if k in testname:
			dis_str = v

	# Read content (detailed log file messages)
	log_str = ''
	with open(log_file, 'r') as f2:
		log_str = f2.read()
		log_str = filter_page(testname, log_str)

	# Write PDF
	c = canvas.Canvas(pdf_file)

	# Write 2 sections
	break_into_pages(line_breaks(" \n {} \n {} \n \n {}".format(dis_str, "_"*125, res_str)), c, 58, 'Summary of data checks', False, 15)
	break_into_pages(line_breaks(log_str), c, 58, 'Detailed log output', True, 10)

	# Save
	c.save()


	return pdf_file

"""
Filter a config object
"""
def filter_config(res_file, testname):
	# Configuration object
	cf = configparser.ConfigParser()
	cf.read(res_file)

	for s in cf.sections():
		# Remove log (unnecessary for CMS)
		if 'log' in cf[s]:
			cf[s].pop('log')

		# Remove section (if not relevant for testXX)
		if not(testname in s):
			cf.remove_section(s)

	# Metadata
	title = os.path.basename(res_file).replace('.ini', '')
	time = 'generated on {}'.format(date.today())

	cf.add_section('metadata')
	cf.set('metadata', 'dataset', title)
	cf.set('metadata', 'test', testname)
	cf.set('metadata', 'time', time)

	# Write config to string
	res_str = StringIO()
	cf.write(res_str)

	return res_str.getvalue().replace('\n', ' \n ')

"""
Filter page (by test name)
"""
def filter_page(s, page_str):
	out_str = ''
	s1 = s.replace('-', '_')
	s2 = s.replace('_', '-')

	for l in page_str.splitlines():	
		iden = l.split(':')[0]

		if 'not implemented yet' in l:
			continue

		if s1 in iden or s2 in iden or s in iden:
			out_str += (l + '\n')
	return out_str

"""
Init PDF page (body)
"""
def init_page(c, leading, font_size=7):

	# Normal text prepare
	t = c.beginText()
	t.setFillColorRGB(0, 0, 0)
	t.setFont('Calibri', font_size)
	t.setCharSpace(0.5)
	t.setTextOrigin(40, 700)
	t.setLeading(leading)
	return t

"""
Init PDF page (title)
"""
def init_title(c, font_size=9):

	# TITLE text prepare - color-c3s: #941333
	t = c.beginText()
	t.setFillColorRGB(0.58, 0.075, 0.2)
	t.setFont('Calibri', font_size)
	t.setCharSpace(0.5)
	t.setTextOrigin(40, 725)
	return t

"""
Write PDF page
"""
def write_page(c, body, title, title_str, leading):

	# Header
	c.drawInlineImage(os.path.join(imgdir, "header.png"), 0, 755, height=1.2*inch, width=8.5*inch)

	# Text (title + body)
	title.textLine(title_str)
	c.drawText(title)
	c.drawText(body)

	# Footer
	c.drawImage(os.path.join(imgdir, "footer.png"), 0, 0, mask='auto', height=1.5*inch, width=8.5*inch)

	# Finalize
	c.showPage()

	return init_title(c), init_page(c, leading)

"""
Get page title/section
"""
def get_page_title(log_page, tit, page_count):

	if log_page:
		return tit + ' (page {})'.format(page_count)
	else:
		return tit

"""
Write a string into pages
"""
def break_into_pages(txtstr, c, lines_per_page, tit, log_page, leading, limit_pages=50):

	title = init_title(c)
	body = init_page(c, leading)
	count_lines = 0
	page_count = 1

	for l in txtstr.split('\n'):
		body.textLine(l)
		if count_lines == lines_per_page:
			title, body = write_page(c, body, title, get_page_title(log_page, tit, page_count), leading)
			count_lines = 0
			page_count += 1
			if page_count == limit_pages:
				break
		else:
			count_lines+=1

	# Last page
	if count_lines != 0:
		write_page(c, body, title, get_page_title(log_page, tit, page_count), leading)

