#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)
#

import os
import glob
import pandas as pd
from bokeh.plotting import figure, output_file, show, save
from bokeh.palettes import Category20

'''
Get download statistics
'''

log_folder = '/data/logs_cds_download'
p1 = figure(title='Requests to CDSAPI (File size vs Download time)', width=1280, height=768, x_axis_label='File size (Mb)', y_axis_label='Request time (s)')
p2 = figure(title='Requests to CDSAPI MEAN(File size vs Download time)', width=1280, height=768, x_axis_label='Mean File size (Mb)', y_axis_label='Request time (s)')
r=0
d=0

file_list = sorted(glob.glob(log_folder + '/*.csv'))
colors = Category20[len(file_list)]

for f in file_list:

    # Loading and aggregating .csv log files from all Virtual Machines
    df = pd.read_csv(f, dtype='object', error_bad_lines=False)

    # Distinguishing errors for avoiding corrupted values
    df_error = df[~df.download_error.isna()]
    df = df[df.download_error.isna()]
    df.file_size = pd.to_numeric(df.file_size.values)
    df.download_time = pd.to_numeric(df.download_time.values)
    df.datetime_request = pd.to_datetime(df.datetime_request, format='%m%d%y-%H%M%S')

    # Adding hourday and download_rate (Mb/s)
    df['hourday'] = df.datetime_request.dt.hour
    df['download_rate'] = df.file_size / (df.download_time * 60)
    df = df[df.download_rate > 0.0]

    # Plot
    p1.circle(df.file_size, df.download_time*60.0, size=2, color=colors[d], legend_label=df.iloc[0].dataset_name)
    p2.square(df.file_size.mean(), df.download_time.mean()*60.0, size=20, color=colors[d], legend_label=df.iloc[0].dataset_name+' - MEAN', alpha=0.75)

    # Individual plot
    p = figure(title='Requests {}(File size vs Download time)'.format(df.iloc[0].dataset_name),
              width=1024, height=768, x_axis_label='File size (Mb)', y_axis_label='Request time (s)')
    p.circle(df.file_size, df['download_rate'], size=2, color=colors[d], legend_label=df.iloc[0].dataset_name)
    output_file(os.path.join(log_folder, '{}.html'.format(df.iloc[0].dataset_name)))
    save(p)

    d += 1
    c = df.dataset_name.count()
    r += c

    print('{} -> {} requests at {} MB/s with an average size of {} in {} sec'.format(
        df.iloc[0].dataset_name, c, df.download_rate.mean(), df.file_size.mean(), df.download_time.mean()))

output_file(os.path.join(log_folder, 'mean_download_rate.html'))
save(p1)
output_file(os.path.join(log_folder, 'absolute_download_rate.html'))
save(p2)

print('TOTAL requests = {}'.format(r))