#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)

import os, sys
import json

from dqc_wrapper.utils.utils_db import *
from dqc_wrapper.utils.utils_files import *
from dqc_wrapper.utils.utils_pdf import *

"""
ECFLOW workflow source

[ ] Task 0 - Init/Create database, insert c3s datasets
[ ] Task 1 - Sync cds-c3s-forms
[ ] Task 2 - Download GRIB files + update database
[ ] Task 3 - DQC checks + update database
[*] Task 4 - PDF generation
[ ] Task 5 - Upload PDFs to drupal
[ ] Task 6 - Generate overview HTMLs
[ ] Task 7 - Delete and free up space
"""
def main(args):

    # Check parameters
    if len(args) != 1:
        print('Usage: python pdf_workflow.py <config_file>')
        return -1
    else:
        try:
            # General configuration
            with open(args[0]) as json_file:
                conf = json.load(json_file)
                dqc_conf_dir = conf['dqc_conf_dir']
                dqc_db_file = conf['dqc_db_file']
                dqc_res_dir = conf['dqc_res_dir']
                dqc_log_dir = conf['dqc_log_dir']
        except:
            print('ERROR: while reading the config file provided')
            exit(-1)

    # Create connection to SQLITE file
    try:
        conn = create_connection(dqc_db_file)
    except:
        print('ERROR: Could not connect to database')
        exit(-1)

    # Generate PDF's taking database content - for every workflow dataset
    for d in select_datasets(conn):

        # Naming and results
        dataset_id, dataset_group = d
        zipfname = os.path.join(dqc_res_dir, dataset_id + '.zip')

        # Read configuration and check mandatory fields in the general section
        conf_path = os.path.join(dqc_conf_dir, dataset_id + '.ini')
        conf = CheckerConfig(conf_path)
        conf.read_config(execute=False)

        # Gather dqc results information
        available_checks = {
            'file_format': 'test0-file_format',
            'standard_compliance_grib': 'test1-standard_compliance_grib',
            'standard_compliance_netcdf': 'test1-standard_compliance_netcdf',
            'temporal_completeness': 'test2-temporal_completeness',
            'temporal_consistency': 'test3-temporal_consistency',
            'spatial_completeness': 'test4-spatial_completeness',
            'spatial_consistency': 'test5-spatial_consistency',
            'valid_ranges': 'test6-valid_ranges'
        }

        # For all the available checks
        files2zip = []
        log_msgs = {}
        logfile = initiate_logging(dqc_log_dir, conf.path_files, dataset_id)
        conf.log_file = logfile

        for checkId, testid in available_checks.items():

            # Init as ok(1), if we encounter undef(0) it is set back
            currentResult = 1
            currentMsg = ""

            # Gather info at file level
            test_res = {}

            # Logging per test
            logger = logging.getLogger(testid)
            lastMsg = ''

            for cinfo in select_filedatachecks(conn, dataset_id, checkId):

                # Input parameters
                fname, checkResult, checkMsg = cinfo

                # Calculate overall result - Priority: error[-1], warn[2], ok[1], undef[0]
                test_res = overall_result(checkId, checkResult, currentResult, checkMsg, currentMsg)

                # Store current result
                currentResult = test_res['res']
                currentMsg = test_res['msg']
                log_msgs[fname] = currentMsg

                # Save log entry
                if checkMsg != "" and checkMsg != lastMsg:
                    if test_res['res'] == 'ok':
                        logger.info(checkMsg)
                    elif test_res['res'] == 'err':
                        logger.error(checkMsg)
                    elif test_res['res'] == 'warn':
                        logger.warning(checkMsg)
                    else:
                        pass
                    lastMsg = checkMsg

            # Store last result in configuration object
            conf.save_config(checkId, test_res, verbose=False)

        # Save INI summary file
        res_file = save_result(conf.config, dqc_res_dir, dataset_id + '.ini')

        # Flush log
        logging_reset()

        # Generate PDFs (requires INI result file + logfile)
        if os.path.exists(res_file) and os.path.exists(logfile):
            files2zip += save_all_pdf_reports(conf, res_file)

        # Include HTML plot
        files2zip += glob.glob(dqc_res_dir+'/'+dataset_id+'-valid_ranges.html')

        # Zip all results (if completed - 6 tests + 1 html)
        if len(files2zip) >= 7:
            zip_results(zipfname, files2zip)

if __name__ == "__main__":
    args = sys.argv[1:]
    args = ['/data/joan/c3s512-wp1-datachecker/dqc_wrapper/conf/checker_workflow_conf.json']
    sys.exit(main(args))