#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)

import os
import glob

from dqc_wrapper.utils.utils_db import *
from dqc_wrapper.utils.utils_forms import get_forms_dates
from dqc_wrapper.utils.utils_files import *

# TODO before filling up -> rm dqc_workflow.db;sqlite3 dqc_workflow.db < create_database.sql
# TODO after fillup up -> dqc_wrapper/scripts/getall*

if __name__ == '__main__':

    # Create connection to SQLITE file
    current_dir = os.path.dirname(os.path.realpath(__file__))
    conn = create_connection(os.path.join(current_dir, 'dqc_workflow.db'))

    # Selected datasets for TEST purposes
    selected = [
        'reanalysis-era5-single-levels-monthly-means-2m_temperature',
        'seasonal-monthly-single-levels-2m_temperature-ecmwf-5',
        'reanalysis-era5-single-levels-2m_temperature',
        'seasonal-original-single-levels-2m_temperature-meteo_france-7'
    ]

    # For every configuration file of DQC (a.k.a dataset)
    conf_dir = '/data/joan/c3s512-wp1-datachecker/dqc_wrapper/QAR_dqc/conf'
    forms_dir = '/data/cds-forms-c3s'

    # Available data checks
    datachecks = [
        'file_format',
        'standard_compliance_grib',
        'standard_compliance_netcdf',
        'temporal_completeness',
        'temporal_consistency',
        'spatial_completeness',
        'spatial_consistency',
        'valid_ranges'
    ]

    for cf in glob.glob(conf_dir + '/*.ini'):

        # Load selected datasets config file
        datasetId = os.path.basename(cf).replace('.ini', '')
        if not(datasetId in selected):
            continue # skip others

        print('Loading {}'.format(cf))
        datasetGroup = 'reanalysis'
        if 'seasonal' in datasetId:
            datasetGroup = 'seasonal'
        insert_dataset(conn, datasetId, datasetGroup)

        # Load config file
        conf = CheckerConfig(cf)
        conf.read_config(execute=False)

        # Get unique sorted list of dates from cds-c3s-forms into the DB
        cds_forms_dates = sorted(list(set(get_forms_dates(conf.cds_dataset, conf.cds_variable, conf.origin, conf.system, forms_dir))))
        files = insert_files(conn, datasetId, cds_forms_dates, conf.cds_variable, conf.origin, conf.system)

        # Initialize data checks
        for f in files:
            for c in datachecks:
                insert_filedatacheck(conn, datasetId, f, c, 0, '')