#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)

import glob
import os
import re

from cdo import Cdo
import datetime as dt

from dqc_wrapper.utils.utils_db import *

'''
Insert into the table filedatetime all the timestamps collected from the GRIB files
'''
def update_filedatetimes(dataset, files):

    cdo = Cdo()

    # Scan for available dates
    for f in files:

        info = cdo.showtimestamp(input=f)

        # Collect all timestamps including HHMMSS
        dates = []
        for inf in info:
            for t in inf.split():
                match = re.search(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}', t)
                if match:  # YYYY-MM-DDTHH:MM:SS minimum requirement
                    dates.append(dt.datetime.strptime(match.group(), '%Y-%m-%dT%H:%M:%S'))

        # Store unique dates
        for d in list(set(dates)):
            insert_filedatetime(conn, d.isoformat(), os.path.basename(f), dataset)


# Create connection to SQLITE file
current_dir = os.path.dirname(os.path.realpath(__file__))
conn = create_connection(os.path.join(current_dir, './dqc_workflow.db'))

# Get all grib files
dataset = 'reanalysis-era5-single-levels-monthly-means-2m_temperature'
files = glob.glob('/shared/cds_downloads/reanalysis/reanalysis-era5-single-levels-monthly-means/2m_temperature/*.grib')
update_filedatetimes(dataset, files)

dataset = 'seasonal-monthly-single-levels-2m_temperature-ecmwf-5'
files = glob.glob('/shared/cds_downloads/seasonal/seasonal-monthly-single-levels/2m_temperature/ecmwf-5*.grib')
update_filedatetimes(dataset, files)
