#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#
#     Authors:
#       Joan Sala Calero (joan.sala@bsc.es)
#

import glob
import os
import pandas as pd

'''
Change stat csv from old format to new
'''
def reformat_csv(old_csv, new_csv):

    # Read file into dataframe
    df_stats = pd.read_csv(old_csv)

    if 'dataType' in df_stats.columns:
        df_stats['File_dt'] = df_stats['File'].map(str) + '_' + df_stats['dataType'].map(str)
    else:
        df_stats['File_dt'] = df_stats['File'].map(str) + '_all'
        df_stats['dataType'] = 'all'

    # Drop old column
    df_stats = df_stats.drop(['File'], axis=1, errors='ignore')

    # Set new index
    df_stats.set_index('File_dt', inplace=True)

    # Remove former index column
    if 'Ind' in df_stats.columns:
        df_stats = df_stats.drop(['Ind'], axis=1, errors='ignore')  # Old datasets might contain this unnecessary column

    # Save file
    df_stats.sort_index(inplace=True)
    df_stats.to_csv(new_csv)
    print(new_csv)
    print('----')

res_dir = '/shared/QAR_dqc/results'
tmp_dir = '/tmp'

for stat_file in sorted(glob.glob('{}/*-stats.csv'.format(res_dir))):

    print(stat_file)
    out_file = os.path.join(tmp_dir, os.path.basename(stat_file))
    reformat_csv(stat_file, out_file)

