#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)

import glob
import os
import re

from cdo import Cdo
import datetime as dt

from dqc_wrapper.utils.utils_db import *

'''
Insert into the table filedatetime all the timestamps collected from the GRIB files
'''
def update_datachecks(dataset, files):

    cdo = Cdo()
    if 'reanalysis' in dataset:
        datachecks_ok = [ 'standard_compliance_netcdf', 'temporal_completeness', 'spatial_completeness', 'spatial_consistency', 'valid_ranges']
        datachecks_warn = [ 'file_format' ]
        datachecks_err = ['standard_compliance_grib', 'temporal_consistency']
    else:
        datachecks_ok = ['temporal_consistency', 'temporal_completeness', 'spatial_completeness', 'spatial_consistency', 'valid_ranges']
        datachecks_warn = ['file_format']
        datachecks_err = ['standard_compliance_grib', 'standard_compliance_netcdf']

    # Scan for available dates
    for f in files:
        for c in datachecks_ok:
            update_filedatacheck(conn, dataset, f, c, 1, '')
        for c in datachecks_err:
            update_filedatacheck(conn, dataset, f, c, -1, '')
        for c in datachecks_warn:
            update_filedatacheck(conn, dataset, f, c, 2, '')
        
# Create connection to SQLITE file
current_dir = os.path.dirname(os.path.realpath(__file__))
conn = create_connection(os.path.join(current_dir, './dqc_workflow.db'))

# Get all grib files
dataset = 'reanalysis-era5-single-levels-monthly-means-2m_temperature'
files = glob.glob('/shared/cds_downloads/reanalysis/reanalysis-era5-single-levels-monthly-means/2m_temperature/*.grib')
update_datachecks(dataset, files)

dataset = 'seasonal-monthly-single-levels-2m_temperature-ecmwf-5'
files = glob.glob('/shared/cds_downloads/seasonal/seasonal-monthly-single-levels/2m_temperature/ecmwf-5*.grib')
update_datachecks(dataset, files)


