#!/usr/bin/env python
# -*- coding: utf-8 -*-

#WRITTEN BY DENE BOWDALO

###------------------------------------------------------------------------------------###
###IMPORT MODULES
###------------------------------------------------------------------------------------###

import crossnetwork_processing_modules
import csv
import datetime
from dateutil.relativedelta import relativedelta
import glob
import numpy as np

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###


def read_data(self):
 
    #headers of metadata/data data to process
    metadata_headers = ['reference','site_name','country','province_state_code','latitude','longitude','altitude','datum','timezone','land_use','sampling_height','meta_year']
    data_headers = ['reference','start_date_UTC','start_time_UTC','end_date_UTC','end_time_UTC','concentration','flag','lod']
    
    #generate metadata/data strutures to store data
    metadata = [[] for x in range(len(metadata_headers))]
    data = [[] for x in range(len(data_headers))]
 
    #get list of all data files
    data_files = glob.glob('/esarchive/obs/uniyork/dene/original_files/%s/CAPMON/*'%(self.species))
    #sort files in a human intuitive order
    data_files = crossnetwork_processing_modules.natsorted(data_files)


    metadata_chunk = False
    read_metadata = False
    data_chunk = False
    read_data = False

    for file_i in range(len(data_files)):
        with open(data_files[file_i], 'rb') as f:
            reader = csv.reader(f,delimiter=',')
            print data_files[file_i]
            
            #get meta process year
            process_year = unicode(data_files[file_i].split('-AllSites-')[-1].split('.csv')[0])
            
            for row in reader:

                #start processing metadata - taking care with french accents
                #find start of metadata chunk
                #use try/except for blank lines
                try:
                    if (row[0] =='*TABLE NAME // NOM DE LA TABLE') & (row[1] == 'Site information'):
                        metadata_chunk = True
                except:
                    pass

                if metadata_chunk == True:
                    #get names of metadata variables 
                    if (row[0] =='*TABLE COLUMN NAME--SHORT FORM // NOM DE COLONNE--FORMAT COURT'):
                        meta_variable_names = row
            
                    #get ready to ready metadata for each site
                    if (crossnetwork_processing_modules.remove_accents(row[0]) == '*TABLE DATA BEGINS // DEBUT DES DONNEES DU TABLEAU'):
                        read_metadata = True
                        continue
                        
                    #stop processing metadata
                    if (crossnetwork_processing_modules.remove_accents(row[0]) == '*TABLE DATA ENDS // FIN DES DONNEES DU TABLEAU'):
                        read_metadata = False
                        metadata_chunk = False
                
                    #read metadata line
                    if read_metadata == True:
                    
                        #check each string in metadata to remove accents
                        #strip leading and trailing whitespace
                        #change all fields to lower case
                        row = [crossnetwork_processing_modules.remove_accents(i).strip().lower() for i in row]
                        
                        #put metadata into lists
                        metadata[metadata_headers.index('reference')].append(row[meta_variable_names.index('SiteID')])
                        metadata[metadata_headers.index('site_name')].append(row[meta_variable_names.index('SiteName')].split('//')[0].strip())
                        metadata[metadata_headers.index('country')].append(row[meta_variable_names.index('CountryCode')].split('(')[-1].split(')')[0].strip())
                        metadata[metadata_headers.index('province_state_code')].append(row[meta_variable_names.index('ProvinceStateCode')])
                        metadata[metadata_headers.index('latitude')].append(row[meta_variable_names.index('Latitude_deg')])
                        metadata[metadata_headers.index('longitude')].append(row[meta_variable_names.index('Longitude_deg')])
                        metadata[metadata_headers.index('altitude')].append(row[meta_variable_names.index('GroundElevAMSL_m')])
                        metadata[metadata_headers.index('datum')].append(row[meta_variable_names.index('LatLonRefDatum')].split('(')[0].strip())
                        metadata[metadata_headers.index('timezone')].append(row[meta_variable_names.index('Timezone')])
                        metadata[metadata_headers.index('land_use')].append(row[meta_variable_names.index('SiteLandUse')].split('//')[0].strip())
                        metadata[metadata_headers.index('sampling_height')].append(row[meta_variable_names.index('SamplingHeightAG')])
                        metadata[metadata_headers.index('meta_year')].append(process_year)
                        
                        #process sampling heights into workable format, as given as time ranges across time record
                    
                        #remove whitespace, remove character 'm', add split string to identify changing metadata, convert back to unicode 
                        metadata[metadata_headers.index('sampling_height')][-1] = unicode(metadata[metadata_headers.index('sampling_height')][-1].replace(" ", "").replace("m","").replace(",","[CHANGINGMETASPLIT]").replace(";","[CHANGINGMETASPLIT]"))
                        #change dates given into YYYYMMDDHHMM format
                        sampling_height_split = metadata[metadata_headers.index('sampling_height')][-1].split('[CHANGINGMETASPLIT]')
                        amended_date_range = []
                        #iterate through different sampling heights for a site
                        for i,var in enumerate(sampling_height_split):
                            date_value_split = var.split(':')
                            #determine if different data ranges are given for a site (otherwise just a number is given)
                            if len(date_value_split) != 1:
                                value = date_value_split[-1]
                                date_split = date_value_split[0].split('-')
                                amended_dates = []
                                #iterate through different dates provided
                                for x,var2 in enumerate(date_split):
                                    if var2 != 'onwards':
                                        if len(var2) == 4:
                                            dd = datetime.datetime.strptime(var2,'%Y')
                                            #if date is the closing year, then add year to have no gaps in datetimes
                                            if x == 1:
                                                dd = dd + relativedelta(years=1)
                                        else:
                                            dd = datetime.datetime.strptime(var2,'%d%b%Y')
                                            #if date is the closing year, then add day to have no gaps in datetimes
                                            if x == 1:
                                                dd = dd + relativedelta(days=1)
                                    else:
                                        dd = 'onwards'
                                    #convert datetimes to output format (YYYYMMDDHHMM)
                                    try:
                                        dd = dd.strftime('%Y%m%d%H%M')
                                    except:
                                        pass
                                    amended_dates.append(dd)
                                amended_date_range.append(amended_dates[0]+'-'+amended_dates[-1]+':'+value)
                        
                        #if date ranges are given then put together in required format (YYYYMMDDHHMM-YYYYMMDDHHMM:value[CHANGINGMETASPLIT]YYYYMMDDHHMM-onwards:value)
                        if len(amended_date_range) != 0:
                            new_sampling_height = ''
                            for n in range(len(amended_date_range)):
                                if n != (len(amended_date_range)-1):
                                    new_sampling_height = new_sampling_height+amended_date_range[n]+'[CHANGINGMETASPLIT]'
                                else:
                                    new_sampling_height = new_sampling_height+amended_date_range[n]
                            metadata[metadata_headers.index('sampling_height')][-1] = unicode(new_sampling_height)
                        
                #start processing data 
                #find start of data chunk
                #use try/except for blank lines   
                try:
                    if (row[0] =='*TABLE NAME // NOM DE LA TABLE') & (row[1] == 'CAPMoN_Ozone'):
                        data_chunk = True
                except:
                    pass
           
                if data_chunk == True:
                    #get names of metadata variables 
                    if (row[0] =='*TABLE COLUMN NAME--SHORT FORM // NOM DE COLONNE--FORMAT COURT'):
                        data_variable_names = row
                    
                    #get ready to ready data for each site
                    if (crossnetwork_processing_modules.remove_accents(row[0]) == '*TABLE DATA BEGINS // DEBUT DES DONNEES DU TABLEAU'):
                        read_data = True
                        continue
                    
                    #stop processing data
                    if (crossnetwork_processing_modules.remove_accents(row[0]) == '*TABLE DATA ENDS // FIN DES DONNEES DU TABLEAU'):
                        read_data = False
                        data_chunk = False
                        
                    #read data line
                    if read_data == True:
                        #put data into lists
                        data[data_headers.index('reference')].append(row[data_variable_names.index('SiteID')])
                        data[data_headers.index('start_date_UTC')].append(row[data_variable_names.index('DateStartUTC')])
                        data[data_headers.index('start_time_UTC')].append(row[data_variable_names.index('TimeStartUTC')])
                        data[data_headers.index('end_date_UTC')].append(row[data_variable_names.index('DateEndUTC')])
                        data[data_headers.index('end_time_UTC')].append(row[data_variable_names.index('TimeEndUTC')])
                        data[data_headers.index('concentration')].append(row[data_variable_names.index('O3_ppbv')])
                        data[data_headers.index('flag')].append(row[data_variable_names.index('O3_ppbv_Flag')])
                        data[data_headers.index('lod')].append(row[data_variable_names.index('O3_ppbv_DetL')])

    self.metadata,self.metadata_headers,self.data,self.data_headers = metadata,metadata_headers,data,data_headers                      