from calendar import monthrange
import copy
import csv
import datetime
from dateutil.relativedelta import relativedelta
from html.parser import HTMLParser
import logging
import struct

import numpy as np
import pandas as pd

from crossnetwork_processing_functions import create_GHOST_stage_logfile, isUTF8
from GHOST_standards import standard_parameters


class ParseAERONETFile(HTMLParser):

    '''class that handles parsing of an AERONET html file'''

    def __init__(self):
        super(ParseAERONETFile, self).__init__(convert_charrefs=True)

        self.have_title = False
        self.have_contact = False
        self.have_column_names = False

    def handle_starttag(self, tag, attrs):
        if tag == 'title':
            self.have_title = True

    def handle_endtag(self, tag):
        pass

    def handle_data(self, data):
        if (self.have_title) & (self.have_contact) & (self.have_column_names):
            data_split = data.strip("\r\n").split(',')
            self.current_n_columns = len(data_split)
            #if number of current line columns is < than number of data columns, do not process line
            if self.current_n_columns >= self.n_columns:
                #if number of current line columns is != to previous number of accepted columns, do not process line
                if (self.previous_n_columns == 0) or (self.previous_n_columns == self.current_n_columns):
                    for column_ii, column_name in enumerate(self.column_names):
                        column_val = data_split[column_ii]
                        if column_name not in ['AERONET_Site','AERONET_Site_Name','Date(dd:mm:yyyy)','Date_(dd:mm:yyyy)','Time(hh:mm:ss)','Time_(hh:mm:ss)','Day_of_Year','Day_of_Year(Fraction)','Last_Processing_Date(dd:mm:yyyy)','Last_Processing_Date','Last_Date_Processed','Last_Processing_Time(hh:mm:ss)','Inversion_Data_Quality_Level','Data_Quality_Level','Retrieval_Measurement_Scan_Type','Measurement_Type(solarorlunar)']:
                            #convert missing measurements to nan
                            column_val = float(column_val)
                            if column_val == -999:
                                column_val = np.NaN
                        self.var_data[column_name].append(column_val)
                    self.previous_n_columns = self.current_n_columns
                    

        elif (self.have_title) & (self.have_contact):
            self.column_names = data.strip("\r\n").replace(" ", "").split(',')
            self.n_columns = len(self.column_names)
            self.previous_n_columns = 0
            self.var_data = {k:[] for k in self.column_names}
            self.have_column_names = True

        elif self.have_title:
            if 'Contact:' in data:
                contact_str = data.strip("\r\n").replace(" ", "")
                #get contact name
                if "PI=" in contact_str:
                    self.contact_name = contact_str.split("PI=")[1].split(";")[0].replace("_", " ")
                #default contact name
                else:
                    self.contact_name = 'Brent Holben'
                #get contact email address
                if "PIEmail=" in contact_str:
                    self.contact_email_address = contact_str.split("PIEmail=")[1].replace("_", " ")
                #default contact email address
                else:
                    self.contact_email_address = 'Brent.N.Holben@nasa.gov'
                #contact institution
                self.contact_institution = 'NASA'
                self.have_contact = True

                #handle case where contact line is not line broken and the column names directly follow
                if "AERONET_Site," in data:
                    column_name_line = data.strip("\r\n").replace(" ", "")
                    column_name_line = 'AERONET_Site,Date'+column_name_line.split('AERONET_Site,Date')[-1]
                    self.column_names = column_name_line.split(',')
                    self.n_columns = len(self.column_names)
                    self.previous_n_columns = 0
                    self.var_data = {k:[] for k in self.column_names}
                    self.have_column_names = True


class ParseBJMEMCFile:

    '''class that handles processing of a BJMEMC file'''

    def __init__(self,filename,run_details):

        self.filename = filename
        self.run_details = run_details
        self.parsed_data = {}

    def read_data(self):

        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')

            for row_count, row in enumerate(csvreader):

                #column headers
                # need to get unique column headers as can have a few station column headers which are repeated 
                # in these cases just keep first instance
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    unique_column_headers = np.unique(column_headers)
                    header_indices = {column_header:column_headers.index(column_header) for column_header in unique_column_headers} 
                    for column_header in unique_column_headers:
                        if column_header not in ['date','hour','type']:
                            self.parsed_data[column_header] = []
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    #make sure number of columns on line is same as number of column headers, if not then skip line
                    if len(row) == len(column_headers): 
                        param = row[header_indices['type']]
                        #only parse data for current parameter
                        if param == self.run_details['parameter']:
                            #parse date
                            date = row[header_indices['date']]
                            year = int(date[:4])
                            month = int(date[4:6])
                            day = int(date[6:])
                            hour = int(row[header_indices['hour']])
                            self.parsed_data['start_dt'].append(datetime.datetime(year=year,month=month,day=day,hour=hour,minute=0)) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                            #parse station measurements
                            for column_header in unique_column_headers:
                                if column_header not in ['date','hour','type']:
                                    self.parsed_data[column_header].append(row[header_indices[column_header]])
                            
            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 


class ParseCANADANAPSFile:

    '''class that handles processing of a CANADA NAPS file'''

    def __init__(self,filename,run_details,NAPS_code,all_NAPS_codes,data_year):

        self.filename = filename
        self.run_details = run_details
        self.parameter = copy.deepcopy(run_details['parameter'])
        self.NAPS_code = NAPS_code
        self.all_NAPS_codes = all_NAPS_codes
        self.data_year = str(data_year)

        #determine file type
        #1. csv?
        if self.filename[-4:] == '.csv':
            self.file_type = 'csv'
        #2. XLS
        elif (self.filename[-4:] == '.XLS') or (self.filename[-4:] == '.xls'):
            self.file_type = 'xls'
            self.data_sheet_names = ['Data']
        #3. xlsx
        elif (self.filename[-5:] == '.xlsx') or (self.filename[-5:] == '.XLSX'):
            self.file_type = 'xlsx' 
            if self.parameter == 'HCHO':
                self.data_sheet_names = ['Carbonyls']
            elif self.parameter in ['CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE']:
                self.data_sheet_names = ['VOC']
            elif self.parameter == 'PM10':
                self.data_sheet_names = ['PM2.5','PM2.5-10']
            elif self.parameter == 'PM2.5':
                self.data_sheet_names = ['PM2.5']
            elif self.parameter in ['PM10_Ca++','PM10_Cl-','PM10_K+','PM10_Mg++','PM10_MSA','PM10_Na+','PM10_NH4+','PM10_NO3-','PM10_SO4--','PM2.5_Ca++','PM2.5_Cl-','PM2.5_K+','PM2.5_Mg++','PM2.5_MSA','PM2.5_Na+','PM2.5_NH4+','PM2.5_NO3-','PM2.5_SO4--']:
                self.data_sheet_names = ['Ions-Spec_IC']
            elif self.parameter in ['PM10_Al','PM10_As','PM10_Cd','PM10_Co','PM10_Cr','PM10_Cu','PM10_Fe','PM10_Mn','PM10_Ni','PM10_Pb','PM10_Se','PM10_V','PM10_Zn','PM2.5_Al','PM2.5_As','PM2.5_Cd','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_Fe','PM2.5_Mn','PM2.5_Ni','PM2.5_Pb','PM2.5_Se','PM2.5_V','PM2.5_Zn']:
                self.data_sheet_names = ['Metals_ICPMS (Near-Total)','Elements_EXDXRF','Elements_EDXRF']
            elif self.parameter in ['PM10_C','PM10_EC','PM10_OC','PM2.5_C','PM2.5_EC','PM2.5_OC']:
                self.data_sheet_names = ['OCEC & OC Artifacts']
            elif self.parameter in ['NH3']:
                self.data_sheet_names = ['Ammonia_IC']
            elif self.parameter in ['HNO3','SO2']:
                self.data_sheet_names = ['Acidic Gaseous PM Precursors_IC']
        self.all_parsed_metadata = []
        self.all_parsed_data = []

        #set dict for helping parse some badly formatted dates
        self.month_dict = {'JAN':'01','FEB':'02','MAR':'03','APR':'04','MAY':'05','JUN':'06','JUL':'07','AUG':'08','SEP':'09','OCT':'10','NOV':'11','DEC':'12'}

    def read_data(self):
       
        #read csv file type
        if self.file_type == 'csv':

            parsed_metadata = {}
            parsed_data = {'start_dt':[],'end_dt':[],'measurement':[],'station_reference':[], 'longitude':[], 'latitude':[], 'station_name':[]}

            start_read = False

            #get encoding of data file
            is_utf8 = isUTF8(self.filename)
            if is_utf8:
                file_encoding = 'UTF-8-SIG'
            else:
                file_encoding = 'ISO-8859-1'

            with open(self.filename, newline='',encoding=file_encoding) as csvfile:
                csvreader = csv.reader(csvfile, delimiter=',', quotechar='"', skipinitialspace=True)
                for row_count, row in enumerate(csvreader):

                    #empty row then continue
                    if len(row) == 0:
                        continue

                    #remove french from header information 
                    if start_read == False:
                        row = [col.split('//')[0].replace(' ','') for col in row]

                    #get file revision date
                    if 'filegeneratedon' in row[0].lower():  
                        revision_date = ','.join(row).lower().replace('filegeneratedon','').replace(',','').replace(':','').strip()
                        if '-' in revision_date:
                            rev_date_split = revision_date.split('-')
                            parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[0]), month=int(rev_date_split[1]), day=int(rev_date_split[2])) 
                        elif '/' in revision_date:
                            rev_date_split = revision_date.split('/')
                            parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[2]), month=int(rev_date_split[1]), day=int(rev_date_split[0])) 
                                
                    #get units
                    if 'units' in row[0].lower():
                        units = ','.join(row).replace('Units','').replace('units','').replace(',','').replace(':','').strip()
                        #replace micro sign if necessary
                        if u'\xb5' in units:
                            units = units.replace(u'\xb5','u')
                        #replace anomalous symbol which sometimes appears
                        if u'\xc2' in units:
                            units = units.replace(u'\xc2','') 
                        parsed_metadata['units'] = units
                        continue
            
                    #get column headers
                    if ('pollutant' in row[0].lower()) & ('H01' in row):
                        start_read = True
                        column_names = copy.deepcopy(row)
                        column_inds = {'date':column_names.index('Date'), 'H01':column_names.index('H01'), 'station_reference':column_names.index('NAPSID'), 'longitude':column_names.index('Longitude'), 'latitude':column_names.index('Latitude'), 'station_name':column_names.index('City')}
                        #check if have method codes
                        if 'Method' in column_names:
                            column_inds['method code'] = column_names.index('Method')
                            parsed_data['method code'] = []
                        elif 'MethodCode' in column_names:
                            column_inds['method code'] = column_names.index('MethodCode')
                            parsed_data['method code'] = []
                        continue

                    if start_read:

                        date = row[column_inds['date']] 
                        if '-' in date:
                            date_split = date.split('-')
                            date_year = date_split[0]
                            date_month = date_split[1]
                            date_day = date_split[2]
                        elif '/' in date:
                            date_split = date.split('/')
                            date_year = date_split[2]
                            date_month = date_split[0]
                            date_day = date_split[1]
                        else:
                            date_year = date[:4]
                            date_month = date[4:6]
                            date_day = date[6:]
                            
                        #iterate through hours
                        for hh in range(24):
          
                            parsed_data['start_dt'].append(datetime.datetime(year=int(date_year), month=int(date_month), day=int(date_day), hour=hh, minute=0))
                            parsed_data['end_dt'].append(parsed_data['start_dt'][-1] + relativedelta(hours=1))
                    
                            #measurements - fill values empty as empty str
                            meas = row[column_inds['H01']+hh]
                            if meas in ['-999','-9999']:
                                parsed_data['measurement'].append('')
                            else:
                                parsed_data['measurement'].append(meas)
            
                            #station reference                  
                            parsed_data['station_reference'].append(row[column_inds['station_reference']])
                            
                            #longitude/latitude
                            parsed_data['longitude'].append(row[column_inds['longitude']])
                            parsed_data['latitude'].append(row[column_inds['latitude']])

                            #station name
                            parsed_data['station_name'].append(row[column_inds['station_name']])

                            #method code
                            if 'method code' in column_inds:
                                parsed_data['method code'].append(row[column_inds['method code']])

                #add to all parsed lists
                self.all_parsed_data.append(parsed_data)
                self.all_parsed_metadata.append(parsed_metadata)   

        #xls/xlsx continuous file types
        elif self.file_type in ['xls','xlsx']:
            if self.file_type == 'xls':
                try:
                    sheets = pd.read_excel(self.filename, sheet_name=None, header=None, na_filter=False)
                except struct.error as err:
                    self.run_details['log_file_str'] += 'FILE IS CORRUPTED. CANNOT READ.\n'
                    return
                    
            elif self.file_type == 'xlsx':
                try:
                    sheets = pd.read_excel(self.filename, sheet_name=None, header=None, na_filter=False, engine='openpyxl')
                except struct.error as err:
                    self.run_details['log_file_str'] += 'FILE IS CORRUPTED. CANNOT READ.\n'
                    return                    

            #get data
            #only 1 sheet, then must be data on it
            if len(sheets.keys()) == 1:
                self.data_sheet_names = [list(sheets.keys())[0]]
                
            #iterate through all data sheets to process
            for data_sheet_name in self.data_sheet_names:
                if data_sheet_name not in sheets:
                    continue
                data_rows = sheets[data_sheet_name].values.tolist()

                parsed_metadata = {'data_sheet_name':data_sheet_name}
                parsed_data = {'start_dt':[],'end_dt':[],'measurement':[]}

                #get metadata
                metadata_sheet_name = ''
                if 'MetaData' in sheets.keys():
                    metadata_sheet_name = 'MetaData'
                elif 'Metadata' in sheets.keys():
                    metadata_sheet_name = 'Metadata'
                elif 'Metadata Information' in sheets.keys():
                    metadata_sheet_name = 'Metadata Information'
                if '/NAPSReferenceMethodPM2.5/' in self.filename:
                    metadata_sheet_name = copy.deepcopy(data_sheet_name)

                sampling_field = ''
                sampler_options = {}
                if metadata_sheet_name != '':
                    metadata_rows = sheets[metadata_sheet_name].values.tolist()
                    for metadata_row in metadata_rows:
                        if len(metadata_row) == 0:
                            continue
                        if '/NAPSReferenceMethodPM2.5/' in self.filename:
                            meta_key = metadata_row[3]
                            meta_value = metadata_row[4]
                        else:
                            meta_key = metadata_row[0]
                            meta_value = metadata_row[1]
                        #convert meta key/values NaNs to empty str
                        if pd.isnull(meta_key):
                            meta_key = '' 
                        else:
                            meta_key = str(meta_key).strip('*').strip().replace('/','_')
                        if pd.isnull(meta_value):
                            meta_value = ''
                        elif type(meta_value) != datetime.datetime:
                            meta_value = str(meta_value).strip()

                        #do variable correction for NAPSReferenceMethodPM2.5 files
                        if 'sampler #' in meta_key.lower():
                            meta_key = 'sampler' 
                            
                        #file revision date
                        if 'file generated' in meta_key.lower():
                            if type(meta_value) == datetime.datetime:
                                parsed_metadata['file_revision_date'] = meta_value
                            else:
                                if '-' in meta_value:
                                    rev_date_split = meta_value.split('-')
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[0]), month=int(rev_date_split[1]), day=int(rev_date_split[2]))
                                elif '/' in meta_value:
                                    rev_date_split = meta_value.split('/')
                                    if int(rev_date_split[2]) < 50:
                                        year_str = '20{}'.format(int(rev_date_split[2]))
                                    else:   
                                        year_str = '19{}'.format(int(rev_date_split[2]))
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(year_str), month=int(rev_date_split[1]), day=int(rev_date_split[0]))

                        #basic station meta
                        elif meta_key.lower() in ['longitude','latitude']:
                            if meta_key.lower() not in parsed_metadata:
                                parsed_metadata[meta_key.lower()] = meta_value
                        elif meta_key.lower() == 'elevation (m)':
                            if 'altitude' not in parsed_metadata:
                                parsed_metadata['altitude'] = meta_value
                        elif meta_key.lower() == 'probe height (m)':
                            if 'sampling_height' not in parsed_metadata:
                                parsed_metadata['sampling_height'] = meta_value
                        elif meta_key.lower() in ['site name','city']:
                            if 'station_name' not in parsed_metadata:
                                parsed_metadata['station_name'] = meta_value
                        elif meta_key.lower() == 'address':
                            if 'address' not in parsed_metadata:
                                parsed_metadata['address'] = meta_value
                        #station classifications
                        elif meta_key.lower() in ['spatial scale','land use','site type']:
                            meta_key_insert = meta_key.lower().replace(' ','_')
                            if meta_key_insert not in parsed_metadata:
                                parsed_metadata[meta_key_insert] = meta_value.split('-')[0].strip()

                        #sampling data - xlsx
                        elif 'Station Info' in sheets.keys():
                            if meta_key == '':
                                sampling_field = ''
                            elif meta_key.lower() in ['sampler','cartridge','medium','observation type','analytical instrument','sample preparation','method code']:
                                sampling_field = meta_key.lower()    
                                sampler_options[sampling_field] = {}
                            elif sampling_field != '':
                                sampler_options[sampling_field][meta_key] = meta_value      
                        #sampling data - xls
                        elif meta_key.lower() in ['sampler','cartridge','medium','observation type','analytical instrument','sample preparation','method code']:
                            if meta_key.lower() in parsed_metadata:
                                parsed_metadata[meta_key.lower()] = '{};{}'.format(parsed_metadata[meta_key.lower()],meta_value)
                            else:
                                parsed_metadata[meta_key.lower()] = meta_value
                
                #get station info
                if 'Station Info' in sheets.keys():
                    stninfo_rows = sheets['Station Info'].values.tolist()
                    sampler_prefix = ''
                    for stninfo_row in stninfo_rows:
                        if len(stninfo_row) == 0:
                            continue
                        meta_key = stninfo_row[0]
                        meta_value = stninfo_row[1]
                        #convert meta key/values NaNs to empty str
                        if pd.isnull(meta_key):
                            meta_key = '' 
                        else:
                            meta_key = meta_key.strip('*').strip()
                        if pd.isnull(meta_value):
                            meta_value = ''
                        elif type(meta_value) != datetime.datetime:
                            meta_value = str(meta_value).strip()
                    
                        #file revision date
                        if 'file generated' in meta_key.lower():
                            if type(meta_value) == datetime.datetime:
                                parsed_metadata['file_revision_date'] = meta_value
                            else:
                                if '-' in meta_value:
                                    rev_date_split = meta_value.split('-')
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[0]), month=int(rev_date_split[1]), day=int(rev_date_split[2]))
                                elif '/' in meta_value:
                                    rev_date_split = meta_value.split('/')
                                    if int(rev_date_split[2]) < 50:
                                        year_str = '20{}'.format(int(rev_date_split[2]))
                                    else:   
                                        year_str = '19{}'.format(int(rev_date_split[2]))
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(year_str), month=int(rev_date_split[1]), day=int(rev_date_split[0]))

                        #basic station meta
                        elif meta_key.lower() in ['longitude','latitude']:
                            if meta_key.lower() not in parsed_metadata:
                                parsed_metadata[meta_key.lower()] = meta_value
                        elif meta_key.lower() == 'elevation (m)':
                            if 'altitude' not in parsed_metadata:
                                parsed_metadata['altitude'] = meta_value
                        elif meta_key.lower() == 'probe height (m)':
                            if 'sampling_height' not in parsed_metadata:
                                parsed_metadata['sampling_height'] = meta_value
                        elif meta_key.lower() in ['site name','city']:
                            if 'station_name' not in parsed_metadata:
                                parsed_metadata['station_name'] = meta_value
                        elif meta_key.lower() == 'address':
                            if 'address' not in parsed_metadata:
                                parsed_metadata['address'] = meta_value
                        #station classifications
                        elif meta_key.lower() in ['spatial scale','land use','site type']:
                            meta_key_insert = meta_key.lower().replace(' ','_')
                            if meta_key_insert not in parsed_metadata:
                                parsed_metadata[meta_key_insert] = meta_value.split('-')[0].strip()

                        #sampler number
                        elif 'Sampler #' in meta_key:
                            sampler_prefix = 'S-{}'.format(meta_key.split('#')[1][0])
                            if 'sampler' not in sampler_options: 
                                sampler_options['sampler'] = {}
                    
                        #sampler instrument name
                        elif (sampler_prefix != '') & (meta_key.lower() == 'description'): 
                            sampler_options['sampler'][sampler_prefix] = meta_value
                            sampler_prefix = ''
            
                #find column names line
                if '/NAPSReferenceMethodPM2.5/' in self.filename:
                    data_column_names_ind = 0
                else:
                    for data_row_ii, data_row in enumerate(data_rows): 
                        if ('NAPS ID' in data_row) or ('NAPS Site ID' in data_row):
                            data_column_names_ind = copy.deepcopy(data_row_ii) 
                            break

                data_column_names = [str(data_column_name) for data_column_name in data_rows[data_column_names_ind]]
                #set inds for desired fields to parse - multiple text variations for each parameter makes it hard work
                #for measurements can have multiple samples, so take first sample preferentially 
                data_column_inds = {}
                vars_to_test = {'station_reference':['NAPS ID','NAPS Site ID'],'date':['Sampling Date','Sample Date','Date','DATE','Sampling Date YYYY/MM/DD'], 'start_time':['Start Time','Start_time','START TIME','START_TIME'], 'end_time':['Stop Time','End Time','End_time','END_TIME'], 'duration':['DURATION'], 'sample_type':['Sample Type','Sampling Type'], 'media':['Media'], 'cartridge':['Cartridge','Cart'], 'pressure':['Pres.','pres.','PRESS','PRESSURE','pressure'], 'temperature':['Temp.','temp.','TEMP','TEMPERATURE','temperature'], 'observation_type':['C/F','Fraction','fraction'], 'measurement':[self.NAPS_code], 'data_flag':self.all_NAPS_codes, 'lod':self.all_NAPS_codes}
                #set measurement units initially as empty str
                units = ''
                for key,vals in vars_to_test.items():
                    for val in vals:
                        if key == 'measurement':
                            if key not in data_column_inds:
                                for data_column_ii, data_column_name in enumerate(data_column_names):
                                    if val == data_column_name:
                                        data_column_inds[key] = copy.deepcopy(data_column_ii) 
                                        break
                                    elif (val in data_column_name) & ('g/m3' in data_column_name):
                                        data_column_inds[key] = data_column_ii
                                        units = data_column_name.split(' ')[-1].strip().strip('()')
                                        break
                                    elif (val in data_column_name) & ('m-3' in data_column_name):
                                        data_column_inds[key] = data_column_ii
                                        units = data_column_name.split(' ')[-1].strip().strip('()')
                                        break
                                if key in data_column_inds:
                                    break 
                        elif key == 'data_flag':
                            if key not in data_column_inds:
                                if '{}-VFlag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-VFlag'.format(val))
                                    break  
                                elif '{}_VFlag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_VFlag'.format(val))
                                    break  
                                elif '{} VFlag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} VFlag'.format(val))
                                    break  
                                elif '{}-Vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-Vflag'.format(val))
                                    break  
                                elif '{}_Vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_Vflag'.format(val))
                                    break  
                                elif '{} Vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} Vflag'.format(val))
                                    break  
                                elif '{}-vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-vflag'.format(val))
                                    break  
                                elif '{}_vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_vflag'.format(val))
                                    break  
                                elif '{} vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} vflag'.format(val))
                                    break  
                        elif key == 'lod':
                            if key not in data_column_inds:
                                if '{}-MDL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-MDL'.format(val))
                                    break  
                                elif '{}-DL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-DL'.format(val))
                                    break  
                                elif '{}_MDL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_MDL'.format(val))
                                    break  
                                elif '{}_DL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_DL'.format(val))
                                    break  
                                elif '{} MDL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} MDL'.format(val))
                                    break  
                                elif '{} DL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} DL'.format(val))
                                    break  
                                elif '{} D.L.'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} D.L.'.format(val))
                                    break  
                                elif '{} D.L'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} D.L'.format(val))
                                    break  
                        else:
                            if key not in data_column_inds:
                                if val in data_column_names:
                                    data_column_inds[key] = data_column_names.index(val)
                                    break     

                #if have not found measurement index, then move on to next data sheet 
                if 'measurement' not in data_column_inds:
                    self.all_parsed_data.append(parsed_data)
                    self.all_parsed_metadata.append(parsed_metadata) 
                    continue

                #if data_flag or lod not filled, make sure it is not stated in next 2 rightward columns
                if 'data_flag' not in data_column_inds:
                    column_start_ii = data_column_inds['measurement']+1
                    column_end_ii = data_column_inds['measurement']+3
                    for data_column_ii, data_column_name in enumerate(data_column_names[column_start_ii:column_end_ii]):
                        if data_column_name == 'VFlag':
                            data_column_inds['data_flag'] = column_start_ii + data_column_ii
                            break
                        elif data_column_name == 'Vflag':
                            data_column_inds['data_flag'] = column_start_ii + data_column_ii
                            break
                        elif data_column_name == 'vflag':
                            data_column_inds['data_flag'] = column_start_ii + data_column_ii
                            break

                if 'lod' not in data_column_inds:   
                    column_start_ii = data_column_inds['measurement']+1
                    column_end_ii = data_column_inds['measurement']+2
                    for data_column_ii, data_column_name in enumerate(data_column_names[column_start_ii:column_end_ii]):
                        if data_column_name == 'MDL':
                            data_column_inds['lod'] = column_start_ii + data_column_ii  
                            break
                        elif data_column_name == 'DL':
                            data_column_inds['lod'] = column_start_ii + data_column_ii 
                            break 
                        elif data_column_name == 'D.L.':
                            data_column_inds['lod'] = column_start_ii + data_column_ii  
                            break
                        elif data_column_name == 'D.L':
                            data_column_inds['lod'] = column_start_ii + data_column_ii     
                            break

                #do little extra work to get units if not already parsed
                #for XLS files --> on first line
                if self.file_type == 'xls':
                    if units == '':
                        #PMSpeciation files do not have any set units so assume they are ug/m3
                        if ('/PMSpeciation/' in self.filename) & (self.filename[-9:].lower() == '_spec.xls'):
                            units = 'ug/m3'
                        else:
                            #if have no set units, assume units for different file types
                            try:    
                                units = data_rows[0][0].split('(')[1].split(')')[0].strip()
                            except IndexError as error:
                                if ('/VOC/' in self.filename):
                                    units = 'ug/m3'
                                else:
                                    units = ''
                    if 'pressure' in data_column_inds:
                        pressure_units = 'kPa'
                    if 'temperature' in data_column_inds:
                        temperature_units = 'degC'
                    if 'lod' in data_column_inds:
                        lod_units = copy.deepcopy(units)
                #for xlsx files --> line above column names
                elif self.file_type == 'xlsx':
                    if units == '':
                        units = data_rows[data_column_names_ind-1][data_column_inds['measurement']]
                    if 'pressure' in data_column_inds:
                        pressure_units = data_rows[data_column_names_ind-1][data_column_inds['pressure']]
                    if 'temperature' in data_column_inds:
                        temperature_units = data_rows[data_column_names_ind-1][data_column_inds['temperature']]
                    if 'lod' in data_column_inds:
                        lod_units = data_rows[data_column_names_ind-1][data_column_inds['lod']]
                        #if lod units are empty str then assume they are same as measured units
                        if lod_units == '':
                            lod_units = copy.deepcopy(units) 
                #add units to parsed_metadata - convert micro sign if neccessary
                if u'\xb5' in units:
                    units = units.replace(u'\xb5','u')
                #replace anomalous symbol which sometimes appears
                if u'\xc2' in units:
                    units = units.replace(u'\xc2','') 
                parsed_metadata['units'] = units
                if 'pressure' in data_column_inds:
                    parsed_metadata['pressure_units'] = pressure_units
                if 'temperature' in data_column_inds:
                    #replace degree symbol if necessary
                    if u'\xb0' in temperature_units:
                        temperature_units = temperature_units.replace(u'\xb0','deg')
                    parsed_metadata['temperature_units'] = temperature_units
                if 'lod' in data_column_inds:
                    if u'\xb5' in lod_units:
                        lod_units = lod_units.replace(u'\xb5','u')
                    if u'\xc2' in lod_units:
                        lod_units = lod_units.replace(u'\xc2','') 
                    parsed_metadata['lod_units'] = lod_units

                #for xlsx files handle sampler and method information on data sheet
                #use sampler_options dict to parse sampler metadata
                if self.file_type == 'xlsx':
                    sampler_fields = ['sampler','cartridge','medium','observation type','analytical instrument','sample preparation','method code']
                    for data_row in data_rows[:data_column_names_ind]:
                        if len(data_row) == 0:
                            continue
                        meta_key = data_row[0]
                        meta_value = data_row[data_column_inds['measurement']]
                        if pd.isnull(meta_key):  
                            meta_key = ''
                        if pd.isnull(meta_value):
                            meta_value = ''  
                        if meta_key.lower() in sampler_fields:
                            if meta_key.lower() not in sampler_options:
                                parsed_metadata[meta_key.lower()] = meta_value
                            else:
                                if meta_value not in sampler_options[meta_key.lower()]:
                                    parsed_metadata[meta_key.lower()] = meta_value
                                else:
                                    parsed_metadata[meta_key.lower()] = sampler_options[meta_key.lower()][meta_value] 
            
                #iterate through rows
                for data_row in data_rows[data_column_names_ind+1:]:

                    if len(data_row) == 0:
                        continue

                    #get date - if missing then skip row
                    date = data_row[data_column_inds['date']]
                    str_date = str(date).strip()
                    if type(date) == int:
                        continue
                    if str_date in ['-999','-9999','','nan']:
                        continue
                    if type(date) == str:
                        archaic_date_format_eval = [True for month in list(self.month_dict.keys()) if month in str_date.upper()] 
                        if len(archaic_date_format_eval) > 0:
                            archaic_date_format = True
                        else:
                            archaic_date_format = False
                        if '-' in str_date:
                            date_split = str_date.split('-')
                            if len(date_split[0]) == 4:
                                year_n = 0
                                month_n = 1
                                day_n = 2
                            elif len(date_split[2]) == 4:
                                year_n = 2
                                month_n = 1
                                day_n = 0
                            elif archaic_date_format:
                                year_n = 2
                                month_n = 1
                                day_n = 0
                            else:
                                year_n = 0
                                month_n = 1
                                day_n = 2  
                            if archaic_date_format:
                                year_str = date_split[year_n]
                                month_str = self.month_dict[date_split[month_n].upper()]
                                day_str = date_split[day_n]
                            else:
                                year_str = date_split[year_n]
                                month_str = date_split[month_n]
                                day_str = date_split[day_n]

                        elif '/' in str_date:
                            date_split = str_date.split('/')
                            if len(date_split[0]) == 4:
                                year_n = 0
                                month_n = 1
                                day_n = 2
                            elif len(date_split[2]) == 4:
                                year_n = 2
                                month_n = 0
                                day_n = 1
                            elif archaic_date_format:
                                year_n = 2
                                month_n = 1
                                day_n = 0
                            else:
                                year_n = 2
                                month_n = 0
                                day_n = 1  
                            if archaic_date_format:
                                year_str = date_split[year_n]
                                month_str = self.month_dict[date_split[month_n].upper()]
                                day_str = date_split[day_n]
                            else:
                                year_str = date_split[year_n]
                                month_str = date_split[month_n]
                                day_str = date_split[day_n]
                        else:
                            continue

                        #test that year, month, day are numbers
                        try:
                            int_year_str = int(year_str)
                            int_month_str = int(month_str)
                            int_day_str = int(day_str)
                        except ValueError as err:
                            continue
 
                        #need to make year 4 characters?
                        if len(year_str) == 2:
                            if int(year_str) < 50:
                                year_str = '20{}'.format(year_str)
                            else:   
                                year_str = '19{}'.format(year_str)  
       
                        #check parsed year is same as data year, otherwise continue
                        if self.data_year != year_str:
                            continue

                        #check date is valid, otherwise continue
                        try:
                            date = datetime.datetime(year=int(year_str),month=int(month_str),day=int(day_str))
                        except ValueError as err:
                            continue

                    #check parsed year is same as data year, otherwise continue
                    if int(self.data_year) != date.year:
                        continue

                    have_time = False

                    #time - have start time and duration
                    #if either of start time or duration are missing then assume date is for whole day (unless stated in filename)
                    if ('start_time' in data_column_inds) & ('duration' in data_column_inds): 
                        start_time = str(data_row[data_column_inds['start_time']]).strip()
                        duration = str(data_row[data_column_inds['duration']]).strip()
    
                        have_time = True                    
                        for test_var in [start_time,duration]:
                            if test_var in ['-999','-9999','','nan','???','?']:
                                have_time = False
                                break

                        if have_time:
                            if ':' in start_time:
                                st_split = start_time.split(':')
                            elif ';' in start_time:
                                st_split = start_time.split(';')
                            elif '.' in start_time:
                                start_time = str(round(float(start_time),2))
                                st_split = start_time.split('.')
                            start_hr = int(st_split[0])
                            start_min = int(st_split[1])
                            if '.' in duration:
                                duration = str(round(float(duration),2))
                                dur_split = duration.split('.')
                                duration_hours = int(dur_split[0])
                                duration_mins = int(dur_split[1])
                            elif ':' in duration:
                                dur_split = duration.split(':')
                                duration_hours = int(dur_split[0])
                                duration_mins = int(dur_split[1])                                
                            elif 'hr' in duration:
                                dur_split = duration.split(' ')
                                duration_hours = int(dur_split[0].replace('hr',''))
                                duration_mins = int(dur_split[1].replace('min',''))
                            else:    
                                duration_hours = int(duration)
                                duration_mins = 0
                            parsed_data['start_dt'].append(date.replace(hour=start_hr,minute=start_min))
                            parsed_data['end_dt'].append(parsed_data['start_dt'][-1] + relativedelta(hours=duration_hours,minutes=duration_mins))  
                                                      
                    #time - have start and end time
                    #if either of start time or end_time are missing then assume date is for whole day (unless stated in filename)
                    elif ('start_time' in data_column_inds) & ('end_time' in data_column_inds):
                        start_time = str(data_row[data_column_inds['start_time']]).strip()
                        end_time = str(data_row[data_column_inds['end_time']]).strip()
                        
                        #if start or end time is empty then continue to next line
                        have_time = True
                        for test_var in [start_time,end_time]:
                            if test_var in ['-999','-9999','','nan']:
                                have_time = False
                                break

                        if have_time:
                            if ':' in start_time:
                                st_split = start_time.split(':')
                            elif ';' in start_time:
                                st_split = start_time.split(';')
                            elif '.' in start_time:
                                start_time = str(round(float(start_time),2))
                                st_split = start_time.split('.')
                            if ':' in end_time:
                                et_split = end_time.split(':')
                            elif ';' in end_time:
                                et_split = end_time.split(';')
                            elif '.' in end_time:
                                end_time = str(round(float(end_time),2))
                                et_split = end_time.split('.')
                            start_hr = int(st_split[0])
                            start_min = int(st_split[1])
                            end_hr = int(et_split[0])
                            end_min = int(et_split[1])
                            parsed_data['start_dt'].append(date.replace(hour=start_hr,minute=start_min)) 
                            if end_hr >= 24:
                                days_to_add = end_hr//24
                                end_hr = end_hr - (days_to_add*24) 
                                parsed_data['end_dt'].append((parsed_data['start_dt'][-1] + relativedelta(days=days_to_add)).replace(hour=end_hr,minute=end_min))
                            else:
                                parsed_data['end_dt'].append(date.replace(hour=end_hr,minute=end_min)) 
                                
                    #no time - so assume daily sample (unless stated in data filename)
                    if not have_time:
                        if '_4hr_' in self.filename:
                            parsed_data['start_dt'].append(date.replace(hour=12))
                            parsed_data['end_dt'].append(date.replace(hour=16))
                        else:
                            parsed_data['start_dt'].append(date)
                            parsed_data['end_dt'].append(parsed_data['start_dt'][-1] + relativedelta(days=1))  

                    #add rest of variables
                    for var in data_column_inds.keys():
                        if var not in ['date','start_time','end_time','duration']:
                            if var not in parsed_data:
                                parsed_data[var] = []
                            val = str(data_row[data_column_inds[var]]).strip()
                            if val in ['-999','-9999','','nan']:
                                parsed_data[var].append('')
                            else:
                                parsed_data[var].append(val)

                #add to all parsed lists
                self.all_parsed_data.append(parsed_data)
                self.all_parsed_metadata.append(parsed_metadata)    

        #convert lists to numpy arrays
        for parsed_data_ii, parsed_data in enumerate(self.all_parsed_data):
            for column_header in list(parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.all_parsed_data[parsed_data_ii][column_header] = np.array(parsed_data[column_header])
                else:
                    self.all_parsed_data[parsed_data_ii][column_header] = np.array(parsed_data[column_header], dtype=np.str) 
     

class ParseCAPMoNFile:

    '''class that handles processing of a CAPMoN file'''

    def __init__(self,filename, parameter, parameter_details):

        self.filename = filename
        self.parameter = parameter
        self.parameter_details = parameter_details

        self.parsed_metadata = {}
        self.parsed_detection_limits = {'siteid':[], 'year':[], 'instrument_detection_limit':[]}
        if self.parameter == 'O3':
            self.parsed_data = {'siteid':[], 'start_dt':[], 'end_dt':[], 'measurements':[], 'flags':[], 'lods':[]}
        else:
            self.parsed_data = {'siteid':[], 'start_dt':[], 'end_dt':[], 'measurements':[], 'flags':[]}
            
        if self.parameter == 'PM10_Co':
            self.lod_ref = 'COB_NGM3'
            self.flag_ref = 'COB_NGM3_Flag'
            self.meas_ref = 'COB_NGM3'
        elif self.parameter == 'PM10_Zn':
            self.lod_ref = 'ZN_NGM3'
            self.flag_ref = 'Zn_NGM3_Flag'
            self.meas_ref = 'Zn_NGM3'
        else:
            self.lod_ref = '{}_NGM3'.format(self.parameter_details['chemical_formula'].upper())
            self.flag_ref = '{}_NGM3_Flag'.format(self.parameter_details['chemical_formula'].upper())
            self.meas_ref = '{}_NGM3'.format(self.parameter_details['chemical_formula'].upper())

    def read_data(self):

        active_meta = False
        active_table = False
        table_name = ''
         
        with open(self.filename, newline='', encoding='latin1') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"', skipinitialspace=True)
            for row_count, row in enumerate(csvreader):
            
                #empty row
                if len(row) == 0:
                    continue

                #remove french translation from row columns
                row = [r.split('//')[0].strip() for r in row]

                #file revision date?
                if row[0] == '*DATE THIS FILE GENERATED':
                    self.parsed_metadata['file_revision_date'] = [row[1]]

                #beginning of data table?
                if row[0] == '*TABLE NAME':
                    #site information
                    if row[1] == 'Site information':
                        table_name = 'site_information'
                    #instrument detection limits
                    elif row[1] == 'Instrument Detection Limits':
                        table_name = 'instrument_detection_limits'
                    #data
                    elif row[1] in ['CAPMoN_Ozone','GLBM METALS']:
                        table_name = 'data'
                    else:
                        table_name = ''

                    if table_name != '':
                        active_meta = True
                    continue

                #start of data table?
                if (row[0] == '*TABLE DATA BEGINS') & (table_name != ''):
                    active_meta = False
                    active_table = True
                    continue

                #end of data table?
                if row[0] == '*TABLE DATA ENDS':
                    active_table = False

                #active table meta?
                if active_meta:
                    #fill codes
                    if row[0] == '*TABLE COLUMN MISSING CODE':
                        missing_codes = row[1:]
                        #transform '-999.' to '-999.0' to correctly catch missing codes
                        missing_codes = ['-999.0' if missing_code == '-999.' else missing_code for missing_code in missing_codes]

                    #column names
                    if row[0] == '*TABLE COLUMN NAME--SHORT FORM': 
                        column_names = row[1:]
                        if table_name == 'site_information':
                            for column_name in column_names:
                                if column_name != '':
                                    if column_name == 'SiteID':
                                        self.parsed_metadata['siteid'] = []
                                    else:
                                        self.parsed_metadata[column_name.lower()] = []  

                #parse data line
                if active_table:
                    row = row[1:]

                    #replace fill values with empty strings
                    for column_ii, missing_code in enumerate(missing_codes):
                        if row[column_ii] == missing_code:
                           row[column_ii] = ''

                    #metadata
                    if table_name == 'site_information':
                        for column_name in column_names:
                            if column_name != '':
                                if column_name == 'SiteName':
                                    self.parsed_metadata[column_name.lower()].append(row[column_names.index(column_name)].split(';')[0])
                                elif column_name == 'SiteID':
                                    self.parsed_metadata['siteid'].append(row[column_names.index(column_name)])
                                else:
                                    self.parsed_metadata[column_name.lower()].append(row[column_names.index(column_name)])

                    elif table_name == 'instrument_detection_limits':
                       #siteid
                        self.parsed_detection_limits['siteid'].append(row[column_names.index('SiteID')])
                        #year
                        self.parsed_detection_limits['year'].append(row[column_names.index('Year')])
                        #detection limit
                        self.parsed_detection_limits['instrument_detection_limit'].append(row[column_names.index(self.lod_ref)])
                        
                        
                    #measurement data
                    elif table_name == 'data':

                        #SiteID 
                        self.parsed_data['siteid'].append(row[column_names.index('SiteID')])

                        #parse datetime
                        start_date = row[column_names.index('DateStartUTC')].split('-')
                        start_time = row[column_names.index('TimeStartUTC')].split(':')
                        end_date = row[column_names.index('DateEndUTC')].split('-')
                        end_time = row[column_names.index('TimeEndUTC')].split(':')

                        #O3
                        if self.parameter == 'O3':
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[0]),month=int(start_date[1]),day=int(start_date[2]),hour=int(start_time[0]),minute=int(start_time[1]),second=int(start_time[2])))
                            self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[0]),month=int(end_date[1]),day=int(end_date[2]),hour=int(end_time[0]),minute=int(end_time[1]),second=int(end_time[2])))
                            self.parsed_data['measurements'].append(row[column_names.index('O3_ppbv')])
                            self.parsed_data['flags'].append(row[column_names.index('O3_ppbv_Flag')])
                            self.parsed_data['lods'].append(row[column_names.index('O3_ppbv_DetL')])
                        #particulate metals
                        else:
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[0]),month=int(start_date[1]),day=int(start_date[2]),hour=int(start_time[0]),minute=int(start_time[1])))
                            self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[0]),month=int(end_date[1]),day=int(end_date[2]),hour=int(end_time[0]),minute=int(end_time[1])))
                            self.parsed_data['measurements'].append(row[column_names.index(self.meas_ref)])
                            self.parsed_data['flags'].append(row[column_names.index(self.flag_ref)])
                         

        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 
        for column_header in list(self.parsed_metadata.keys()):
            self.parsed_metadata[column_header] = np.array(self.parsed_metadata[column_header], dtype=np.str) 
        for column_header in list(self.parsed_detection_limits.keys()):
            self.parsed_detection_limits[column_header] = np.array(self.parsed_detection_limits[column_header], dtype=np.str)
        

class ParseCHILESINCAFile:

    '''class that handles processing of a CHILE SINCA file'''

    def __init__(self,filename):

        self.filename = filename
        temporal_resolution = self.filename.split('_')[-1].split('.')[0]
        if temporal_resolution == 'hourly':
            self.timedelta = relativedelta(hours=1)
        elif temporal_resolution == 'daily':
            self.timedelta = relativedelta(days=1)
        elif temporal_resolution == 'monthly':
            self.timedelta = relativedelta(months=1)

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=';', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers if column_header not in ['FECHA (YYMMDD)','HORA (HHMM)']}
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        if column_header == 'FECHA (YYMMDD)':
                            line_year = element[:2]
                            if int(line_year) >= 50:
                                line_year = '19'+line_year
                            else:
                                line_year = '20'+line_year
                            line_month = element[2:4]
                            line_day = element[4:6] 
                            line_hour = row[column_headers.index('HORA (HHMM)')][:2]
                            line_minute = row[column_headers.index('HORA (HHMM)')][2:4]
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(line_year),month=int(line_month),day=int(line_day),hour=int(line_hour),minute=int(line_minute))) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + self.timedelta)
                        elif column_header != 'HORA (HHMM)':
                            element = element.replace(',','.')
                            self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 


class ParseCNEMCFile:

    '''class that handles processing of a CNEMC file'''

    def __init__(self,filename,run_details):

        self.filename = filename
        self.run_details = run_details

    def read_data(self):

        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')

            for row_count, row in enumerate(csvreader):

                #column headers
                # need to get unique column headers as can have a few station column headers which are repeated 
                # in these cases just keep first instance
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    unique_column_headers = np.unique(column_headers)
                    header_indices = {column_header:column_headers.index(column_header) for column_header in unique_column_headers} 
                    self.parsed_data = {column_header:[] for column_header in unique_column_headers if column_header not in ['date','hour','type']}
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    #make sure number of columns on line is same as number of column headers, if not then skip line
                    if len(row) == len(column_headers): 
                        param = row[header_indices['type']]
                        #only parse data for current parameter
                        if param == self.run_details['parameter']:
                            #parse date
                            date = row[header_indices['date']]
                            year = int(date[:4])
                            month = int(date[4:6])
                            day = int(date[6:])
                            hour = int(row[header_indices['hour']])
                            self.parsed_data['start_dt'].append(datetime.datetime(year=year,month=month,day=day,hour=hour,minute=0)) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                            #parse station measurements
                            for column_header in unique_column_headers:
                                if column_header not in ['date','hour','type']:
                                    self.parsed_data[column_header].append(row[header_indices[column_header]])
                            
            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 


class ParseEANETFile:

    '''class that handles processing of a EANET file'''

    def __init__(self,filename):

        self.filename = filename

    def read_data(self):

        with open(self.filename, newline='', encoding="cp850") as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #parameter units
                if row_count == 0:
                    row_units = [unit.strip().encode("ascii", "ignore").decode() for unit in row]
                #column headers
                elif row_count == 1:
                    column_headers = [column_header.strip().encode("ascii", "ignore").decode() for column_header in row]

                    #correct certain column header names 
                    to_correct = {'Ca2+':'Ca++','Mg2+':'Mg++','SO42-':'SO4--'}
                    for to_c in to_correct.keys():
                        if to_c in column_headers:
                            column_headers[column_headers.index(to_c)] = to_correct[to_c]

                    #check if date/time is repeated if so, modify column names
                    if column_headers.count('Date') >= 2:
                        column_headers[column_headers.index('Date')] = 'Start_Date'
                        column_headers[column_headers.index('Time')] = 'Start_Time'
                        column_headers[column_headers.index('Date')] = 'End_Date'
                        column_headers[column_headers.index('Time')] = 'End_Time'
                    self.parsed_data = {column_header:[] for column_header in column_headers if column_header not in ['Start_Date','Start_Time','End_Date','End_Time','Year','Month','Day','Hour']}
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []
                    self.units = {column_header:row_unit for row_unit,column_header in zip(row_units,column_headers) if (column_header not in ['Country','Site','Start_Date','Start_Time','End_Date','End_Time','Year','Month','Day','Hour','Memo','Note']) & (row_unit != 'Unit')}
                #data lines
                else:
                    #remove non-ascii characters from row
                    row = [element.strip().encode("ascii", "ignore").decode() for element in row]

                    #do not append line if row all blank, or if have missing key time variables
                    all_blank = all('' == r for r in row)
                    if all_blank:
                        continue

                    #check and process time information       
                    #non-automatic time
                    if 'Start_Date' in column_headers:
                        if (row[column_headers.index('Start_Date')] == '') or (row[column_headers.index('End_Date')] == ''):
                            continue
                        start_date_split = row[column_headers.index('Start_Date')].split('/')
                        end_date_split = row[column_headers.index('End_Date')].split('/')
                        start_time_split = row[column_headers.index('Start_Time')].split(':')
                        end_time_split = row[column_headers.index('End_Time')].split(':')

                        #no start date column, skip line
                        if (len(start_date_split) == 1) & (start_date_split[0] == ''):
                            continue
                        #no month or day, assume XXXX:01:01
                        elif len(start_date_split) == 1:
                            line_year_start = start_date_split[0]
                            line_month_start = '01'
                            line_day_start = '01'
                        #no day, assume XXXX:XX:00
                        elif len(start_date_split) == 2:
                            line_year_start = start_date_split[0]
                            line_month_start = start_date_split[1]
                            line_day_start = '01'
                        #all date information present
                        else:
                            line_year_start = start_date_split[0]
                            line_month_start = start_date_split[1]
                            line_day_start = start_date_split[2] 

                        #no end date column, skip line
                        if (len(end_date_split) == 1) & (end_date_split[0] == ''):
                            continue
                        #no month or day, assume XXXX:01:01
                        elif len(end_date_split) == 1:
                            line_year_end = end_date_split[0]
                            line_month_end = '01'
                            line_day_end = '01'
                        #no day, assume XXXX:XX:00
                        elif len(end_date_split) == 2:
                            line_year_end = end_date_split[0]
                            line_month_end = end_date_split[1]
                            line_day_end = '01'
                        #all date information present
                        else:
                            line_year_end = end_date_split[0]
                            line_month_end = end_date_split[1]
                            line_day_end = end_date_split[2] 

                        #no start time column, assume 00:00:00
                        if (len(start_time_split) == 1) & (start_time_split[0] == ''):
                            line_hour_start = '00'
                            line_minute_start = '00'
                            line_second_start = '00'
                        #no minutes or seconds, assume XX:00:00
                        elif len(start_time_split) == 1:
                            line_hour_start = start_time_split[0]
                            line_minute_start = '00'
                            line_second_start = '00'
                        #no seconds, assume XX:XX:00
                        elif len(start_time_split) == 2:
                            line_hour_start = start_time_split[0]
                            line_minute_start = start_time_split[1]
                            line_second_start = '00'
                        #all time information present
                        else:
                            line_hour_start = start_time_split[0]
                            line_minute_start = start_time_split[1]
                            line_second_start = start_time_split[2] 

                        #no end time column, assume 00:00:00
                        if (len(end_time_split) == 1) & (end_time_split[0] == ''):
                            line_hour_end = '00'
                            line_minute_end = '00'
                            line_second_end = '00'
                        #no minutes or seconds, assume XX:00:00
                        elif len(end_time_split) == 1:
                            line_hour_end = end_time_split[0]
                            line_minute_end = '00'
                            line_second_end = '00'
                        #no seconds, assume XX:XX:00
                        elif len(end_time_split) == 2:
                            line_hour_end = end_time_split[0]
                            line_minute_end = end_time_split[1]
                            line_second_end = '00'
                        #all time information present
                        else:
                            line_hour_end = end_time_split[0]
                            line_minute_end = end_time_split[1]
                            line_second_end = end_time_split[2]    

                        #check date/time is sensible before generating datetimes, if not do not process line
                        if (len(line_year_start) != 4) or (len(line_month_start) > 2) or (len(line_day_start) > 2) or (len(line_hour_start) > 2) or (len(line_minute_start) > 2) or (len(line_second_start) > 2):
                            continue
                        if (len(line_year_end) != 4) or (len(line_month_end) > 2) or (len(line_day_end) > 2) or (len(line_hour_end) > 2) or (len(line_minute_end) > 2) or (len(line_second_end) > 2):
                            continue

                        self.parsed_data['start_dt'].append(datetime.datetime(year=int(line_year_start),month=int(line_month_start),day=int(line_day_start),hour=int(line_hour_start),minute=int(line_minute_start),second=int(line_second_start)))
                        self.parsed_data['end_dt'].append(datetime.datetime(year=int(line_year_end),month=int(line_month_end),day=int(line_day_end),hour=int(line_hour_end),minute=int(line_minute_end),second=int(line_second_end)))

                    #automatic time
                    else:
                        #if have no year then do not append line
                        if row[column_headers.index('Year')] == '':
                            continue
                        #if have no month then assume measurement is annual average
                        elif 'Month' not in column_headers:
                            line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=1,day=1,hour=0,minute=0)
                            line_end_dt = line_start_dt + relativedelta(years=1)
                        elif row[column_headers.index('Month')] == '':
                            line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=1,day=1,hour=0,minute=0)
                            line_end_dt = line_start_dt + relativedelta(years=1)
                        #if have no day then assume measurement is monthly average
                        elif 'Day' not in column_headers:
                            line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=1,hour=0,minute=0)
                            line_end_dt = line_start_dt + relativedelta(months=1)
                        elif row[column_headers.index('Day')] == '':
                            line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=1,hour=0,minute=0)
                            line_end_dt = line_start_dt + relativedelta(months=1)
                        #if have no hour then assume measurement is daily average
                        elif 'Hour' not in column_headers:
                            line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=int(row[column_headers.index('Day')]),hour=0,minute=0)
                            line_end_dt = line_start_dt + relativedelta(days=1)
                        elif row[column_headers.index('Hour')] == '':
                            line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=int(row[column_headers.index('Day')]),hour=0,minute=0)
                            line_end_dt = line_start_dt + relativedelta(days=1)
                        #otherwise have hourly data
                        else:
                            line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=int(row[column_headers.index('Day')]),hour=int(row[column_headers.index('Hour')])-1,minute=0)
                            line_end_dt = line_start_dt + relativedelta(hours=1)                       

                        #do check to ensure datetimes are within a sensible range (1900-2100)
                        if (line_start_dt < datetime.datetime(1900,1,1)) or (line_end_dt < datetime.datetime(1900,1,1)) or (line_start_dt > datetime.datetime(2100,1,1)) or (line_end_dt > datetime.datetime(2100,1,1)):
                            continue

                        self.parsed_data['start_dt'].append(line_start_dt)
                        self.parsed_data['end_dt'].append(line_end_dt)

                    #append rest of desired line values
                    for column_header, element in zip(column_headers, row):
                        if column_header in list(self.parsed_data.keys()):
                            self.parsed_data[column_header].append(element) 

        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str)


class ParseEEAAIRBASEFile:

    '''class that handles processing of a EEA AIRBASE file'''

    def __init__(self,filename):

        self.filename = filename
        filename_split = self.filename.split('/')[-1]
        self.component_code = filename_split[7:12]
        self.measurement_european_group_code = filename_split[12:17]
        self.temporal_resolution = filename_split[17:].split('.')[0]
        self.data_file_start_date = datetime.datetime(int(filename_split.split('.')[-2].split('-')[2]), int(filename_split.split('.')[-2].split('-')[1]), int(filename_split.split('.')[-2].split('-')[0])) 
        self.data_file_end_date = datetime.datetime(int(filename_split.split('.')[-1].split('-')[2]), int(filename_split.split('.')[-1].split('-')[1]), int(filename_split.split('.')[-1].split('-')[0]))

        if 'hour' in self.temporal_resolution:
            if len(self.temporal_resolution) > 4:
                self.n_steps = int(self.temporal_resolution[:-4])
            else:
                self.n_steps = 1
            self.file_format = 'column'
            self.timedelta = relativedelta(hours=self.n_steps)
        elif 'day' in self.temporal_resolution:
            if len(self.temporal_resolution) > 3:
                self.file_format = 'row' 
            else:
                self.file_format = 'column'
                self.timedelta = relativedelta(days=1)
        elif 'week' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'month' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'year' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'var' in self.temporal_resolution:
            self.file_format = 'row'

        self.column_headers = ['start_dt','end_dt','measurement','flag']
        self.parsed_data = {column_header:[] for column_header in self.column_headers}

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter='\t', quotechar='"')
 
            for row_count, row in enumerate(csvreader):

                if self.file_format == 'column':
                    date = row[0]
                    base_dt = datetime.datetime(year=int(date[:4]),month=int(date[5:7]),day=int(date[8:10]),hour=0,minute=0)

                    measurement_inds = np.arange(1,len(row),2,dtype=int)
                    flag_inds = np.arange(2,len(row),2,dtype=int)

                    for step_ii,measurement_ind in enumerate(measurement_inds):
                        self.parsed_data['start_dt'].append(base_dt)
                        self.parsed_data['end_dt'].append(base_dt + self.timedelta)
                        base_dt = copy.deepcopy(self.parsed_data['end_dt'][-1])
                        self.parsed_data['measurement'].append(row[measurement_ind])
                    for flag_ind in flag_inds:
                        self.parsed_data['flag'].append(row[flag_ind])

                elif self.file_format == 'row':
                    start_date = row[0]
                    end_date = row[1]
                    self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[:4]),month=int(start_date[5:7]),day=int(start_date[8:10]),hour=int(start_date[11:13]),minute=int(start_date[14:16])))
                    self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[:4]),month=int(end_date[5:7]),day=int(end_date[8:10]),hour=int(end_date[11:13]),minute=int(end_date[14:16])))
                    self.parsed_data['measurement'].append(row[2])
                    self.parsed_data['flag'].append(row[3])

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 


class ParseJAPANNIESFile:

    '''class that handles processing of a JAPAN NIES file'''

    def __init__(self,filename,file_type):

        self.filename = filename
        self.file_type = file_type
        
        if self.file_type == 'hourly':
            self.file_type = 'hourly'
            self.column_dict = {'year':0,'station_reference':1,'species':3,'units':4,'month':5,'day':6,'hour1':7}
            self.column_headers = ['station_reference','species','units','start_dt','end_dt','measurement']
        elif self.file_type == 'monthly': 
            self.file_type = 'monthly'
            self.column_dict = {'year':0,'species':3,'station_reference':11,'measured_days':35,'measured_hours':47,'april':59}
            self.column_headers = ['station_reference','species','units','start_dt','end_dt','measured_hours_percent','measurement']
            self.units = {'SO2':'PPM','NO':'PPM','NO2':'PPM','CO':'PPM','OX':'PPM','NMHC':'PPMC','CH4':'PPMC','THC':'PPMC','SPM':'MG/M3','PM25':'UG/M3'}


        self.parsed_data = {column_header:[] for column_header in self.column_headers}

    def read_data(self):

        with open(self.filename, newline='', encoding='cp932') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',',quotechar='"') 
            for row_count, row in enumerate(csvreader):
                
                if row_count > 0:

                    #strip row
                    row = [x.strip() for x in row]

                    #hourly files
                    if self.file_type == 'hourly':
                        
                        #first check if current day exists 
                        try:
                            dt_day = datetime.datetime(year=int(row[self.column_dict['year']]), month=int(row[self.column_dict['month']]), day=int(row[self.column_dict['day']]), hour=0, minute=0)
                        #if not then move to next line
                        except ValueError:
                            continue
                    
                        #iterate through hours
                        for hh in range(24):
                            #time
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(row[self.column_dict['year']]), month=int(row[self.column_dict['month']]), day=int(row[self.column_dict['day']]), hour=hh, minute=0))
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                            
                            #measurements
                            meas = row[self.column_dict['hour1']+hh]
                            #set measurement fill values (9997,9998,9999,'') as NaN values
                            if meas in ['9997','9998','9999','']:
                                self.parsed_data['measurement'].append(np.NaN)
                            else:
                                self.parsed_data['measurement'].append(meas)
                    
                            #process other variables
                            for var in ['station_reference','species','units']:
                                self.parsed_data[var].append(row[self.column_dict[var]])

                    #monthly files
                    elif self.file_type == 'monthly':

                        #iterate through months
                        for mm_ii, mm in enumerate([4,5,6,7,8,9,10,11,12,1,2,3]):
                            #time
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(row[self.column_dict['year']]), month=mm, day=1, hour=0, minute=0))
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(months=1))

                            #process misc. variables
                            for var in ['station_reference','species']:
                                self.parsed_data[var].append(row[self.column_dict[var]])

                            #add units --> not in file but add information from metadata file
                            self.parsed_data['units'].append(self.units[self.parsed_data['species'][-1]])

                            #measurements
                            if self.parsed_data['species'][-1] in ['NMHC','CH4','THC','PM25']:
                                meas = row[self.column_dict['measured_hours']+mm_ii]
                            else:
                                meas = row[self.column_dict['april']+mm_ii]
                            #set measurement fill values (9997,9998,9999,'') as NaN values
                            if meas in ['9997','9998','9999','']:
                                self.parsed_data['measurement'].append(np.NaN)
                            else:
                                self.parsed_data['measurement'].append(meas)

                            #measured hours --> calculate percent in month
                            if self.parsed_data['species'][-1] in ['NMHC','CH4','THC','PM25']:
                                measured_hours = row[self.column_dict['measured_days']+mm_ii]
                                if (measured_hours != '') & (self.parsed_data['species'][-1] == 'PM25'):
                                    measured_hours = float(measured_hours) * 24.0
                            else:
                                measured_hours = row[self.column_dict['measured_hours']+mm_ii]

                            if measured_hours == '':
                                measured_hours_percent = ''
                            else:
                                measured_hours = float(measured_hours)
                                n_hours_in_month = monthrange(int(row[self.column_dict['year']]),mm)[1]*24.0
                                measured_hours_percent = (100.0/n_hours_in_month)* measured_hours
                            self.parsed_data['measured_hours_percent'].append(measured_hours_percent)

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                elif column_header == 'measurement':
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.float32)
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str)                  


class ParseMEXICOCDMXFile:

    '''class that handles processing of a MEXICO CDMX file'''

    def __init__(self,filename):

        self.filename = filename
        self.start_read = False

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row[0] == 'date':
                    column_headers = copy.deepcopy(row)
                    headers_to_alter = {'cve_station':'id_station','cve_parameter':'id_parameter','date':'start_dt'}
                    column_headers = [headers_to_alter[column_header] if column_header in list(headers_to_alter.keys()) else column_header for column_header in column_headers]
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                    self.parsed_data['end_dt'] = []
                    self.start_read = True
                #data lines
                elif self.start_read:
                    for column_header, element in zip(column_headers, row):
                        if column_header == 'start_dt':
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(element[6:10]),month=int(element[3:5]),day=int(element[:2]),hour=int(element[11:13])-1,minute=int(element[14:16]))) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                        else:
                            self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 


class ParseMITECOFile:

    '''class that handles processing of a MITECO file'''

    def __init__(self,filename,parameter_details):

        #add passed variables to self
        self.filename = filename
        self.parameter_details = parameter_details

        #get file type
        if '_HH_' in self.filename:
            self.file_type = 'hourly'
        elif '_DD_' in self.filename:
            self.file_type = 'daily'
        elif '_Irreg_' in self.filename:
            self.file_type = 'irregular'

        #initialise fields to store data columns
        self.punto_muestreo = []
        self.start_dt = []
        self.temporal_resolutions = []
        self.measurements = []

    def read_data(self):

        with open(self.filename) as f:
            self.content = f.readlines()
            self.content = [x.strip('\n') for x in self.content]

            column_headers = self.content[0].split(';') 

            for row in self.content[1:]:
                data_line = row.split(';')

                #if species code on line is not correct for species processing, move to next line
                if self.parameter_details['miteco_code'] != data_line[column_headers.index('MAGNITUD')]:
                    continue             

                #get time/temporal resolution/measurement
                if self.file_type == 'hourly':
                    n_valid_measurements = 0
                    first_measurement_index = column_headers.index('H01')
                    for hour in range(24):
                        try:
                            dt = datetime.datetime(int(data_line[column_headers.index('ANNO')]),int(data_line[column_headers.index('MES')]),int(data_line[column_headers.index('DIA')]),hour,0)
                            meas = data_line[first_measurement_index+hour] 
                            self.start_dt.append(dt)
                            self.temporal_resolutions.append(60)    
                            self.measurements.append(meas)
                            n_valid_measurements+=1
                        except (ValueError, IndexError) as err:
                            continue

                elif self.file_type == 'daily':
                    n_valid_measurements = 0
                    first_measurement_index = column_headers.index('D01')
                    for day in range(1,32):
                        try:
                            dt = datetime.datetime(int(data_line[column_headers.index('ANNO')]),int(data_line[column_headers.index('MES')]),day,0,0)
                            meas = data_line[first_measurement_index+(day-1)]
                            self.start_dt.append(dt)
                            self.temporal_resolutions.append(60*24)
                            self.measurements.append(meas)
                            n_valid_measurements+=1
                        except (ValueError, IndexError) as err:  
                            continue

                elif self.file_type == 'irregular':  
                    n_valid_measurements = 0
                    try:
                        start_date = data_line[column_headers.index('FECHA_INICIAL')]
                        end_date = data_line[column_headers.index('FECHA_FINAL')]
                        if len(start_date) == 19:
                            start_yyyy = int(start_date[:4])
                            start_mm = int(start_date[5:7])
                            start_dd = int(start_date[8:10])
                            start_hour = int(start_date[11:13]) 
                            start_min = int(start_date[14:16])
                            start_sec = int(start_date[17:19])
                        elif len(start_date) == 10:
                            start_yyyy = int(start_date[:4])
                            start_mm = int(start_date[5:7])
                            start_dd = int(start_date[8:10])
                            start_hour = 0 
                            start_min = 0
                            start_sec = 0
                        else:
                            print(start_date)
                            1+'a'

                        if len(end_date) == 19:
                            end_yyyy = int(end_date[:4])
                            end_mm = int(end_date[5:7])
                            end_dd = int(end_date[8:10])
                            end_hour = int(end_date[11:13]) 
                            end_min = int(end_date[14:16])
                            end_sec = int(end_date[17:19])
                        elif len(end_date) == 10:
                            end_yyyy = int(end_date[:4])
                            end_mm = int(end_date[5:7])
                            end_dd = int(end_date[8:10])
                            end_hour = 0 
                            end_min = 0
                            end_sec = 0
                        else:
                            print(end_date)
                            1+'a'

                        start_dt = datetime.datetime(start_yyyy,start_mm,start_dd,start_hour,start_min,start_sec)
                        end_dt = datetime.datetime(end_yyyy,end_mm,end_dd,end_hour,end_min,end_sec)
                        dt_window = end_dt - start_dt
                        res = int(dt_window.total_seconds()/60.)
                        meas = data_line[column_headers.index('VALOR')]
                        self.start_dt.append(start_dt)
                        self.temporal_resolutions.append(res)
                        self.measurements.append(meas)
                        n_valid_measurements = 1
                    
                    except IndexError:
                        pass

                #if have no valid measurements, move to next line
                if n_valid_measurements == 0:
                    continue
                    
                #get punto_muestreo
                self.punto_muestreo.extend([data_line[column_headers.index('PUNTO_MUESTREO')]]*n_valid_measurements)

        #convert lists to numpy arrays
        self.punto_muestreo = np.array(self.punto_muestreo, dtype=np.str)
        self.start_dt = np.array(self.start_dt)
        self.temporal_resolutions = np.array(self.temporal_resolutions, dtype=np.int)
        self.measurements = np.array(self.measurements, dtype=np.str)

class ParseNASAAmesFile:

    '''class that handles processing of a nasa ames file'''

    def __init__(self,filename,station_reference,run_details):
        
        self.filename = filename
        self.station_reference = station_reference
        self.run_details = run_details

        #core header meta
        self.n_header = ''          #number of header lines
        self.ffi = ''               #file format index
                                    #1st digit indicates the number of independent variables (dimensions)
                                    #2nd digit indicates number of necessary auxiliary variable
                                    #Remaining differences cover the way the independent variables increment and the nature (numeric or character) of the variables
        self.originator_name = ''   #data originator name
        self.originator_affil = ''  #data originator affiliation
        self.provider_name = ''     #data provider name
        self.campaign_name = ''     #campaign name
        self.volume_number = ''     #current volume number (dated field - should be 1)
        self.n_volumes = ''         #number of volumes in dataset (dated field - should be 1)
        self.date_start = ''        #date for which record starts
        self.file_creation = ''     #date of file creation


        #independent variable meta (e.g. time)
        self.independent_interval = ''  #independent variable interval (if changes should be zero)
        self.independent_name = ''      #independent variable name (e.g. days from file reference point)


        #primary variable meta (quantities that have been observed or derived)
        self.n_primary = ''               #number of primary variables
        self.primary_scale_factors = ''   #scale factors for each of primary variables
        self.primary_missing_values = ''  #value imposed for missing data for each primary variable
        self.primary_names = []           #primary variable names (e.g. ozone)
        self.primary_units = []           #primary variable units
        self.primary_details = []         #extra details about variables (e.g. limit of detection)

        #auxiliary variable meta
        self.n_auxiliary = ''    #number of auxiliary variables

        #comments meta
        self.n_comments = ''     #number of lines of comments
        self.comments = []       #extra meta comments about data. Can also include column headings

        #important line counts
        self.endline_primary_count = -999
        self.start_auxiliary = -999
        self.start_comments = -999
        self.start_data = -999
        self.row_count = 0

    def read_header(self):

        with open(self.filename) as f:
            self.content = f.readlines()
            self.content = [x.strip('\n') for x in self.content]

            for row in self.content:

                #--read core header meta--
                if self.row_count == 0:
                    line_split = row.split(' ')
                    self.n_header,self.ffi = line_split[0],line_split[1]

                elif self.row_count == 1:
                    self.originator_name = row

                elif self.row_count == 2:
                    self.originator_affil = row

                elif self.row_count == 3:
                    self.provider_name = row

                elif self.row_count == 4:
                    self.campaign_name = row

                elif self.row_count == 5:
                    line_split = row.split(' ')
                    self.volume_name, self.n_volumes = line_split[0],line_split[1]

                elif self.row_count == 6:
                    line_split = row.split(' ')
                    self.date_start, self.file_creation = line_split[0]+line_split[1]+line_split[2], line_split[3]+line_split[4]+line_split[5]

                #--read independent variable meta--
                elif self.row_count == 7:
                    self.independent_interval = row

                elif self.row_count == 8:
                    self.independent_name = row

                #--read primary variable meta--
                elif self.row_count == 9:
                    self.n_primary = int(row)
                    self.endline_primary_count = 11+self.n_primary
                    self.start_auxiliary = self.endline_primary_count+1

                elif self.row_count == 10:
                    self.primary_scale_factors = row.split(' ')

                elif self.row_count == 11:
                    self.primary_missing_values = row.split(' ')

                elif (self.row_count > 11) & (self.row_count <= self.endline_primary_count):
                    #handle line splitting with quotes (with commas inside)
                    line_split = []
                    split_by_quotes = row.split('"')
                    for split_ii, split_element in enumerate(split_by_quotes):
                        if (split_ii % 2) == 0:
                            line_split = np.append(line_split, split_element.split(','))
                        else:
                            if split_element[:7] == 'Comment':
                                new_str = 'Comment="{}"'.format(split_element[8:])
                            else: 
                                new_str = '{}"{}"'.format(line_split[-1], split_element)
                                line_split = line_split[:-1]
                            line_split = np.append(line_split, new_str)
                    line_split = [element.strip() for element in line_split]
                    line_split = [element for element in line_split if element != '']
                    if len(line_split) == 0:
                        line_split = [row]

                    #only read line if first value is one of accepted primary variable names
                    #otherwise line is appended to primary details - this deals with badly formatted files
                    #also add 1 to auxiliary start count - so don't start reading too soon

                    #create a list of all potentially valid primary variables
                    valid_primary_variables = ['end_time of measurement','numflag']
                    for param in list(standard_parameters.keys()):
                        ebas_parameter_names = standard_parameters[param]['ebas_parameter_name']
                        for ebas_parameter_name in ebas_parameter_names:
                            #only process variables with a valid EBAS parameter name (i.e. not empty string)
                            if ebas_parameter_name != '':
                                valid_primary_variables=np.append(valid_primary_variables,ebas_parameter_name)
                                #create numflag variable for all parameters
                                valid_primary_variables=np.append(valid_primary_variables,'numflag {}'.format(ebas_parameter_name))                    

                    if line_split[0] in valid_primary_variables:
                        self.primary_names.append(line_split[0])
                        self.primary_units.append(line_split[1])
                        if len(line_split) > 2:
                            self.primary_details.append(line_split[2:])
                        else:
                            self.primary_details.append([])
                    else:
                        self.primary_details[-1] = self.primary_details[-1]+line_split
                        self.endline_primary_count+=1
                        self.start_auxiliary+=1

                #--read auxiliary variable meta--
                #first checks if row is a valid number or relates to the previous primary details not correctly formatted
                #if so then append to primary details
                elif self.row_count == self.start_auxiliary:
                    try:
                        self.n_auxiliary = int(row)
                        #if is not 0 then break processing and panic - as have never seen this be used and don't know format
                        if self.n_auxiliary != 0:
                            self.run_details['log_file_str'] += 'Using auxilary variables. {} of them! How Strange. Now need to code for this. In file: {} .Stopping processing.'.format(self.n_auxiliary, self.filename)
                            create_GHOST_stage_logfile(2,self.run_details)
                        self.start_comments = self.start_auxiliary+1
                    except:
                        line_split = row.split(',')
                        self.primary_details[-1] = np.append(self.primary_details[-1],line_split)
                        self.endline_primary_count+=1
                        self.start_auxiliary+=1

                #--read comments meta--
                elif self.row_count == self.start_comments:
                    self.n_comments = int(row)
                    self.end_comments = self.start_comments+self.n_comments
                    self.start_data = self.end_comments+1

                elif (self.row_count > self.start_comments) & (self.row_count <= self.end_comments):
                    self.comments.append(row)

                #manually iterate row count
                self.row_count+=1

    def read_data(self):
        #cut file read to just be data columns
        data = self.content[self.start_data:]

        #make list of lists to store data columns (1 independent variable + primary variables)
        self.structured_data = [[] for _ in range(self.n_primary+1)]

        #determine if any multiplication by scale factors is necessary
        primary_scale_factor_conversion = [True if float(sf) != 1.0 else False for sf in self.primary_scale_factors]

        #iterate through lines, appending column data into a list of lists
        for line in data:
            line_split = line.split()
            for n,var in enumerate(line_split):
                #not independent data variable?
                if n > 0:
                    #convert missing values to NaNs if necessary
                    if var == self.primary_missing_values[n-1]:
                        var = np.NaN
                    #do multiplication by scale factors if necessary (if non NaN)
                    if primary_scale_factor_conversion[n-1]:
                        try:
                            var = float(var)
                        except:
                            var = np.NaN
                        if np.isfinite(var):
                            var = var * float(self.primary_scale_factors[n-1])
                #append
                self.structured_data[n].append(var)


"""A dictionary, where key is the ISD identifier in ASCII and value
is a list that contains [variable_name, number_of_chars]. The variable
name is as in ISD format. The number of chars indicate the number of
characters data (without considering the 3 chars of the identifier"""


isd_map = {'AA1': ['LIQUID-PRECIPITATION', 8],
           'AA2': ['LIQUID-PRECIPITATION', 8],
           'AA3': ['LIQUID-PRECIPITATION', 8],
           'AA4': ['LIQUID-PRECIPITATION', 8],
           'AB1': ['LIQUID-PRECIPITATION MONTHLY TOTAL', 7],
           'AC1': ['PRECIPITATION-OBSERVATION-HISTORY', 3],
           'AD1': ['LIQUID-PRECIPITATION GREATEST AMOUNT IN 24 HOURS, FOR THE MONTH', 19],
           'AE1': ['LIQUID-PRECIPITATION, NUMBER OF DAYS WITH SPECIFIC AMOUNTS, FOR THE MONTH', 12],
           'AG1': ['PRECIPITATION-ESTIMATED-OBSERVATION', 4],
           'AH1': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH2': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH3': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH4': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH5': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH6': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI1': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI2': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI3': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI4': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI5': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI6': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AJ1': ['SNOW-DEPTH', 14],
           'AK1': ['SNOW-DEPTH-GREATEST-ON-GROUND', 12],
           'AL1': ['SNOW-ACCUMULATION', 7],
           'AL2': ['SNOW-ACCUMULATION', 7],
           'AL3': ['SNOW-ACCUMULATION', 7],
           'AL4': ['SNOW-ACCUMULATION', 7],
           'AM1': ['SNOW-ACCUMULATION-GREATEST-IN-24H', 18],
           'AN1': ['SNOW-ACCUMULATION-FOR-MONTH', 9],
           'AO1': ['LIQUID-PRECIPITATION', 8],
           'AO2': ['LIQUID-PRECIPITATION', 8],
           'AO3': ['LIQUID-PRECIPITATION', 8],
           'AO4': ['LIQUID-PRECIPITATION', 8],
           'AP1': ['15MIN-LIQUID-PRECIP', 6],
           'AP2': ['15MIN-LIQUID-PRECIP', 6],
           'AP3': ['15MIN-LIQUID-PRECIP', 6],
           'AP4': ['15MIN-LIQUID-PRECIP', 6],
           'AT1': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],             # Weather Occurrence Data
           'AT2': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT3': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT4': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT5': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT6': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT7': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT8': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AU1': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU2': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU3': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU4': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU5': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU6': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU7': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU8': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU9': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AW1': ['PRESENT-WEATHER-OBSERVATION-AUTOMATED', 3],
           'AW2': ['PRESENT-WEATHER-OBSERVATION-AUTOMATED', 3],
           'AW3': ['PRESENT-WEATHER-OBSERVATION-AUTOMATED', 3],
           'AX1': ['PAST-WEATHER-OBSERVATION', 6],
           'AX2': ['PAST-WEATHER-OBSERVATION', 6],
           'AX3': ['PAST-WEATHER-OBSERVATION', 6],
           'AX4': ['PAST-WEATHER-OBSERVATION', 6],
           'AX5': ['PAST-WEATHER-OBSERVATION', 6],
           'AX6': ['PAST-WEATHER-OBSERVATION', 6],
           'AY1': ['PAST-WEATHER-OBSERVATION manual', 5],
           'AY2': ['PAST-WEATHER-OBSERVATION manual', 5],
           'AZ1': ['PAST-WEATHER-OBSERVATION automated', 5],
           'AZ2': ['PAST-WEATHER-OBSERVATION automated', 5],
           'CB1': ['Subhourly Liquid Precipitation Section', 10],       # Climate Reference Network Unique Data
           'CB2': ['Subhourly Liquid Precipitation Section', 10],
           'CF1': ['Hourly Fan Speed Section', 6],
           'CF2': ['Hourly Fan Speed Section', 6],
           'CF3': ['Hourly Fan Speed Section', 6],
           'CG1': ['PRIMARY-SENSOR', 8],
           'CG2': ['PRIMARY-SENSOR', 8],
           'CG3': ['PRIMARY-SENSOR', 8],
           'CH1': ['HOURLY-RELATIVE-HUMIDITY', 15],
           'CH2': ['HOURLY-RELATIVE-HUMIDITY', 15],
           'CI1': ['RELATIVE-HUMIDITY-TEMPERATURE', 28],
           'CN1': ['HOURLY-BATTERY-VOLTAGE', 18],
           'CN2': ['HOURLY-DIAGNOSTIC', 18],
           'CN3': ['SECONDARY-HOURLY-DIAGNOSTIC', 16],
           'CN4': ['SECONDARY-HOURLY-DIAGNOSTIC', 19],
           'CO1': ['US-NETWORK-METADATA', 5],                           # Network Metadata
           'CO2': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO3': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO4': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO5': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO6': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO7': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO8': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO9': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CR1': ['CONTROL-SECTION', 7],
           'CT1': ['SUBHOURLY-TEMPERATURE', 7],
           'CT2': ['SUBHOURLY-TEMPERATURE', 7],
           'CT3': ['SUBHOURLY-TEMPERATURE', 7],
           'CU1': ['HOURLY-TEMPERATURE', 13],
           'CU2': ['HOURLY-TEMPERATURE', 13],
           'CU3': ['HOURLY-TEMPERATURE', 13],
           'CV1': ['HOURLY-TEMPERATURE-EXTREME', 26],
           'CV2': ['HOURLY-TEMPERATURE-EXTREME', 26],
           'CV3': ['HOURLY-TEMPERATURE-EXTREME', 26],
           'CW1': ['SUBHOURLY-WETNESS', 14],
           'CX1': ['HOURLY-GEONOR-VIBRATING-WIRE', 26],
           'CX2': ['HOURLY-GEONOR-VIBRATING-WIRE', 26],
           'CX3': ['HOURLY-GEONOR-VIBRATING-WIRE', 26],
           'ED1': ['RUNWAY-VISUAL-RANGE', 8],                           # Runway Visual Range Data
           'GA1': ['SKY-COVER-LAYER', 13],                              # Cloud and Solar Data
           'GA2': ['SKY-COVER-LAYER', 13],
           'GA3': ['SKY-COVER-LAYER', 13],
           'GA4': ['SKY-COVER-LAYER', 13],
           'GA5': ['SKY-COVER-LAYER', 13],
           'GA6': ['SKY-COVER-LAYER', 13],
           'GD1': ['SKY-COVER-SUMMATION', 12],
           'GD2': ['SKY-COVER-SUMMATION', 12],
           'GD3': ['SKY-COVER-SUMMATION', 12],
           'GD4': ['SKY-COVER-SUMMATION', 12],
           'GD5': ['SKY-COVER-SUMMATION', 12],
           'GD6': ['SKY-COVER-SUMMATION', 12],
           'GE1': ['SKY-CONDITION', 19],
           'GF1': ['SKY-CONDITION', 23],
           'GG1': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG2': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG3': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG4': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG5': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG6': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GH1': ['HOURLY-SOLAR-RADIATION', 28],
           'GJ1': ['SUNSHINE', 5],
           'GK1': ['SUNSHINE-OBSERVATION', 4],
           'GL1': ['SUSHINE-OBSERVATION-FOR-MONTH', 6],
           'GM1': ['SOLAR-IRRADIANCE', 30],
           'GN1': ['SOLAR-RADIATION', 28],
           'GO1': ['NET-SOLAR-RADIATION', 19],
           'GP1': ['MODELED-SOLAR-IRRADIANCE', 31],
           'GQ1': ['HOURLY-SOLAR-ANGLE', 14],
           'GR1': ['HOURLY-EXTRATERRESTRIAL-RADIATION', 14],
           '???': ['HAIL', 4],                                          # Hail Data - Missing ID in the Doc
           'IA1': ['GROUND-SURFACE', 3],                                # Ground Surface Data
           'IA2': ['GROUND-SURFACE', 9],
           'IB1': ['HOURLY-SURFACE', 27],
           'IB2': ['HOURLY-SURFACE', 13],
           'IC1': ['GROUND-SURFACE', 25],
           'KA1': ['EXTREME-AIR-TEMPERATURE', 10],                      # Temperature Data
           'KA2': ['EXTREME-AIR-TEMPERATURE', 10],
           'KA3': ['EXTREME-AIR-TEMPERATURE', 10],
           'KA4': ['EXTREME-AIR-TEMPERATURE', 10],
           'KB1': ['AVERAGE-AIR-TEMPERATUER', 10],
           'KB2': ['AVERAGE-AIR-TEMPERATUER', 10],
           'KB3': ['AVERAGE-AIR-TEMPERATUER', 10],
           'KC1': ['EXTREME-AIR-TEMP-FOR-MONTH', 14],
           'KC2': ['EXTREME-AIR-TEMP-FOR-MONTH', 14],
           'KD1': ['HEATING-COOLING-DEGREE-DAYS', 9],
           'KD2': ['HEATING-COOLING-DEGREE-DAYS', 9],
           'KE1': ['EXTREME-TEMPS-EXCEEDING-CRITERIA', 12],
           'KF1': ['HOURLY-CALCULATED-TEMP', 6],
           'KG1': ['AVERAGE-DEW_POINT', 11],
           'KG2': ['AVERAGE-DEW_POINT', 11],
           'MA1': ['ATMOSPHERIC-PRESSURE-OBSERVATION', 12],             # Pressure Data
           'MD1': ['ATMOSPHERIC-PRESSURE-CHANGE', 11],
           'ME1': ['GEOPOTENTIAL-HEIGHT', 6],
           'MF1': ['ATMOSPHERIC-PRESSURE-STP-SLP', 12],
           'MG1': ['ATMOSPHERIC-PRESSURE', 12],
           'MH1': ['ATMOSPHERIC-PRESSURE-FOR-MONTH', 12],
           'MK1': ['ATMOSPHERIC-PRESSURE-FOR-MONTH', 24],
           'MV1': ['PRESENT-WEATHER-IN-VICINITY', 3],                   # Weather Occurrence Data (Again?)
           'MV2': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV3': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV4': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV5': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV6': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV7': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MW1': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW2': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW3': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW4': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW5': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW6': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW7': ['PRESENT-WEATHER-OBSERVATION', 3],
           'OA1': ['SUPPLEMENTARY-WIND-OBSERVATION', 8],                # Wind Data
           'OA2': ['SUPPLEMENTARY-WIND-OBSERVATION', 8],
           'OA3': ['SUPPLEMENTARY-WIND-OBSERVATION', 8],
           'OB1': ['Hourly/Sub-Hourly Wind Section', 28],
           'OB2': ['Hourly/Sub-Hourly Wind Section', 28],
           'OC1': ['WIND-GUST-OBSERVATION', 5],
           'OD1': ['SUPPLEMENTARY-WIND-OBSERVATION', 11],
           'OD2': ['SUPPLEMENTARY-WIND-OBSERVATION', 11],
           'OD3': ['SUPPLEMENTARY-WIND-OBSERVATION', 11],
           'RH1': ['RELATIVE HUMIDITY', 9],
           'RH2': ['RELATIVE HUMIDITY', 9],
           'RH3': ['RELATIVE HUMIDITY', 9],
           'SA1': ['SEA-SURFACE-TEMPERATURE', 5],                       # Sea Surface Temperature Data
           'ST1': ['SOIL-TEMPERATURE', 17],                             # Soil Temperature Data
           'UA1': ['WAVE-MEASUREMENT', 10],                             # Marine Data
           'UG1': ['WAVE-MEASUREMENT-PRIMARY', 9],
           'UG2': ['WAVE-MEASUREMENT-SECONDARY', 9],
           'WA1': ['PLATFORM-ICE-ACCRETION', 6],
           'WD1': ['WATER-SURFACE-ICE', 20],
           'WG1': ['WATER-SURFACE-ICE-HISTORICAL', 11],
           'WJ1': ['WATER-LEVEL-OBSERVATION', 19]}


class ISDLineParser:

    def __init__(self, filename, row):

        self.filename = filename
        self.prec_period = None
        self.add_offset = 105  # always fixed
        self.section_chars = 3

        self.varmap = isd_map

        # Include all variables that should appear to every file in __init__ for readability
        # Initialize all to None, then call parsing methods

        self.num_char = None

        # Control section attributes
        self.station_idu = None
        self.station_idn = None
        self.date_ascii = None
        self.time_ascii = None
        self.data_source_flag = None
        self.latitude = None
        self.longitude = None
        self.code = None
        self.elevation = None
        self.call_letter = None
        self.qc_name = None

        # Mandatory section attributes
        # wind-obdervation
        self.direction_angle = None
        self.direction_angle_qc = None
        self.type_code = None
        self.speed = None
        self.speed_qc = None

        # sky-condition
        self.ceiling = None
        self.ceiling_qc = None
        self.ceiling_dc = None

        # visibility
        self.distance = None
        self.distance_qc = None
        self.variability = None
        self.variability_qc = None

        # air-temperature"
        self.air_temp = None
        self.air_temp_qc = None

        # air-temperature dew point"
        self.dpoint_temp = None
        self.dpoint_temp_qc = None

        # atmospheric pressure
        self.sea_level_pressure = None
        self.sea_level_pressure_qc = None

        #non-mandatory field existence checks
        self.have_prec_depth = False
        self.have_cloud = False
        
        # Call the first method to start the initilization of the attributes
        # self.open_file()
        self.parse_file_line(row)

        # Remarks Section
        self.rem_content = None

        # Original Observation Section
        self.has_qnn = False

    def parse_file_line(self, line):
        # First read how many chars the line has
        self.num_char = int(line[0:4]) + self.add_offset
        # Read control and mandatory section
        self.parse_control_section(line)
        self.parse_mandatory_section(line)

        # Additional Data Section
        additional = line[105:108]
        pos = 105
        if additional == "ADD":
            # try to read
            pos += self.section_chars
            while pos < self.num_char:
                try:
                    read = self.parse_add_section(line, pos)
                    pos += read
                except Exception as e:
                    # reached REM
                    # logging.exception(e)
                    break
        while pos < self.num_char:
            if line[pos:pos + self.section_chars] == 'REM':
                try:
                    rem_text = self.parse_rem_section(line[pos:self.num_char])
                    pos += len(rem_text)
                except Exception as e:
                    break
            else:
                break
        if line[pos:pos + self.section_chars] == 'EQD':
            # while pos < num_char:
            try:
                read = self.parse_eqd_section(line[pos:self.num_char])
                pos += read
            except Exception as e:
                print(line[pos:pos + self.section_chars])  # change to logging
        if line[pos:pos + self.section_chars] == 'QNN':
            self.parse_qnn_section()
            print(pos, self.num_char)
        # if pos < self.num_char:


    def parse_control_section(self, line):
        """Function to read the Control Data Section"""

        # scaling factors
        scale_lats = 1000

        self.station_idu = line[4:10]  # station, USAF catalog identifier
        self.station_idn = line[10:15]  # station, NCEI catalog identifier
        self.date_ascii = line[15:23]  # date of observation
        self.time_ascii = line[23:27]  # time on UTC
        self.datetime_ascii = line[15:27]
        self.data_source_flag = line[27]  
        self.latitude = np.float64(line[28:34]) / scale_lats
        if self.latitude == 99.999:
            self.latitude = np.NaN
        self.longitude = np.float64(line[34:41]) / scale_lats  # 000000 to 179999 are signed negative
        if self.longitude == 999.999:
            self.longitude = np.NaN
        self.code = line[41:46]  # type of geophysical surface observation
        self.elevation = np.float32(line[46:51])  # relative to MSL
        if self.elevation == 9999:
            self.elevation = np.NaN
        # self.call_letter = line[51 - 56]  # no idea
        # self.qc_name = line[56:60]  # quality control process name

    def parse_mandatory_section(self, line):
        """Function to read the Mandatory Data Section"""

        # scaling factors
        scale_speed = 10
        scale_temp = 10
        scale_slp = 10
        
        # constant to convert to kelvin
        kelvin = 273.15

        "Wind-Observation"
        self.direction_angle = float(line[60:63])
        self.direction_angle_qc = line[63]
        if self.direction_angle == 999:
            self.direction_angle_qc = '3'
        self.speed = float(line[65:69]) / scale_speed
        self.speed_qc = line[69]
        if self.speed == 999.9:
            self.speed_qc = '3'

        "Sky-Condition-Observation"
        self.ceiling = float(line[70:75])
        self.ceiling_qc = line[75]
        if self.ceiling == 99999:
            self.ceiling_qc = '3'
        if self.ceiling == 22000:
            self.ceiling_qc = 'CHM'

        "Visibility-Observation"
        self.distance = float(line[78:84])
        self.distance_qc = line[84]
        if self.distance == 999999 or self.distance_qc == 9999:
            self.distance_qc = '3'
        if self.distance == 160000:
            self.distance_qc = 'VIM'

        "Air-Temperature-Observation"
        self.air_temp = (float(line[87:92]) / scale_temp) + kelvin
        self.air_temp_qc = line[92]
        if self.air_temp == 999.9:
            self.air_temp_qc = '3'

        "Air-Temperature-Observation dew point"
        self.dpoint_temp = (float(line[93:98]) / scale_temp) + kelvin
        self.dpoint_temp_qc = line[98]
        if self.dpoint_temp == 999.9:
            self.dpoint_temp_qc = '3'

        "Atmospheric-Pressure-Observation"
        self.sea_level_pressure = float(line[99:104]) / scale_slp
        self.sea_level_pressure_qc = line[104]
        if self.sea_level_pressure == 9999.9:
            self.sea_level_pressure_qc = '3'

    def parse_add_section(self, line, position):
        """ Function to parse the Additional Data Section"""

        identifier = line[position:position + self.section_chars]

        # where the section of data start and end according to the id
        start_block = position + self.section_chars
        end_block = start_block + isd_map[identifier][1]
        block = line[start_block:end_block]

        # the data from the additional section
        # call function find_add_id
        self.find_add_id(identifier, block)

        # send back how many characters were read
        read = isd_map[identifier][1] + self.section_chars
        return read

    def find_add_id(self, identifier, block, print_info=False, print_block=False):
        """ According to the input identifier, the function reads respectively
        the variables under the identifier, following the isd format document.
        Unfortunately, this is by far a not elegant way. Suggestions are welcome."""

        map_value = 0  # position of the name in isd_map
        # scale factor - similar for all values thankfully
        scale_factor = 10
        kelvin = 273.15

        #precipitation accumulation (AO prioritised over AA)
        #both have 4 repeating fields , keep just the first field (will always have the highest frequency of measurement)
        #AO     
        if identifier == 'AO1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.prec_period_ao = float(block[0:2]) # already in minutes
            if self.prec_period_ao == 0:
                return
            self.prec_depth_ao = float(block[2:6]) / scale_factor
            self.prec_depth_ao_qc = block[7]
            if self.prec_depth_ao == 999.9:
                self.prec_depth_ao_qc = '3'

            #check if prec_depth is set, if not set it.
            if not self.have_prec_depth:
                self.prec_depth =    copy.deepcopy(self.prec_depth_ao)
                self.prec_period =   copy.deepcopy(self.prec_period_ao)
                self.prec_depth_qc = copy.deepcopy(self.prec_depth_ao_qc)
                self.have_prec_depth = True
            #else if ao qc code is not suspect/erroneous or data missing, overwrite prec_depth  
            elif (self.prec_depth_ao_qc not in ['2','3','6','7']) & (self.prec_depth_ao != 999.9):
                self.prec_depth =    copy.deepcopy(self.prec_depth_ao)
                self.prec_period =   copy.deepcopy(self.prec_period_ao)
                self.prec_depth_qc = copy.deepcopy(self.prec_depth_ao_qc)
        #AA     
        if identifier == 'AA1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.prec_period_aa = float(block[0:2]) * 60  # because we want minutes
            if self.prec_period_aa == 0:
                return
            self.prec_depth_aa = float(block[2:6]) / scale_factor
            self.prec_depth_aa_qc = block[7]
            if self.prec_depth_aa == 999.9:
                self.prec_depth_aa_qc = '3'

            #check if prec_depth is set, if not set it.
            if not self.have_prec_depth:
                self.prec_depth =    copy.deepcopy(self.prec_depth_aa)
                self.prec_period =   copy.deepcopy(self.prec_period_aa)
                self.prec_depth_qc = copy.deepcopy(self.prec_depth_aa_qc)
                self.have_prec_depth = True
            #else if currently set prec_depth qc code is suspect/erroneous or data is missing, and this is not the case with the aa data, overwrite prec_depth  
            elif (self.prec_depth_qc in ['2','3','6','7']) or (self.prec_depth == 999.9): 
                if (self.prec_depth_aa_qc not in ['2','3','6','7']) & (self.prec_depth_aa != 999.9):
                    self.prec_depth =    copy.deepcopy(self.prec_depth_aa)
                    self.prec_period =   copy.deepcopy(self.prec_period_aa)
                    self.prec_depth_qc = copy.deepcopy(self.prec_depth_aa_qc)

        #snow accumulation
        #4 repeating fields, keep just the first field (will always have the highest frequency of measurement)
        elif identifier == 'AL1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.snow_acc_period = float(block[0:2])  * 60  # because we want minutes
            if self.snow_acc_period == 0:
                return
            self.snow_acc = float(block[2:5])
            self.snow_acc_qc = block[6]
            if self.snow_acc == 999:
                self.snow_acc_qc = '3'

        #snow depth
        elif identifier == 'AJ1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.si = float(block[0:4])
            self.si_qc = block[5]
            if self.si == 9999:
                self.si_qc = '3'

        #relative humidity
        #2 repeating fields, keep just the first field (will always have the highest frequency of measurement)
        elif identifier == 'CH1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.avg_rh_period = float(block[0:2])  # already in minutes
            self.avg_rh = float(block[9:13]) / scale_factor
            self.avg_rh_qc = block[13]
            if self.avg_rh == 999.9:
                self.avg_rh_qc = '3'

        #atmospheric pressure
        elif identifier == 'MA1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.station_p = float(block[6:11]) / scale_factor
            self.station_p_qc = block[11]
            if self.station_p == 9999.9:
                self.station_p_qc = '3'

        #cloud coverage (GA prioritised over GD)
        #GA - coverage of specific cloud layers
        #GD - summary of coverage below specific heights
        #both have 6 repeating fields, keep the field with the maximum valid value

        #GA - coverage provided as oktas
        elif identifier in ('GA1', 'GA2', 'GA3', 'GA4', 'GA5', 'GA6'):
            #first cloud layer
            if identifier == 'GA1':
                self.cloud_ga = int(block[0:2])
                self.cloud_ga_qc = block[2]
                #coverage is not in oktas or missing?
                if self.cloud_ga in [9,10,99]:
                    self.cloud_ga = 99
                    self.cloud_ga_qc = '3'
                
            #have higher coverage value in other layer? 
            elif identifier in ('GA2', 'GA3', 'GA4', 'GA5', 'GA6'):
                new_value = int(block[0:2])
                new_qc = block[2]
                #coverage is not in oktas or missing?
                if new_value in [9,10,99]:
                    new_value = 99
                    new_qc = '3'

                #if current cloud_ga_qc is erroneous/suspect or data is missing, and this is not the case for the new data, then overwrite cloud_ga
                if (self.cloud_ga_qc in ['2','3','6','7']) or (self.cloud_ga == 99): 
                    if (new_qc not in ['2','3','6','7']) & (new_value != 99):
                        self.cloud_ga =    copy.deepcopy(new_value)
                        self.cloud_ga_qc = copy.deepcopy(new_qc)
                #else if both current cloud_ga_qc and new_qc are not erroneous/suspect and data is not missing, then check if new coverage is > cloud_ga
                elif (self.cloud_ga_qc not in ['2','3','6','7']) & (self.cloud_ga != 99) & (new_qc not in ['2','3','6','7']) & (new_value != 99):
                    if new_value > self.cloud_ga:
                        self.cloud_ga = copy.deepcopy(new_value)
                        self.cloud_ga_qc = copy.deepcopy(new_qc)

            #check if cloud is set, if not set it 
            if not self.have_cloud:
                self.cloud =    copy.deepcopy(self.cloud_ga)
                self.cloud_qc = copy.deepcopy(self.cloud_ga_qc)
                if self.cloud == 99:
                    self.cloud_frac = 999
                    self.cloud_frac_qc = '3'
                else:
                    self.cloud_frac = (100.0/8.0)*self.cloud
                    self.cloud_frac_qc = copy.deepcopy(self.cloud_ga_qc)
                self.have_cloud = True
            #else if ga qc code is not suspect/erroneous or data missing, overwrite cloud 
            elif self.cloud_ga_qc not in ['2','3','6','7'] & (self.cloud_ga != 99):
                self.cloud =    copy.deepcopy(self.cloud_ga)
                self.cloud_qc = copy.deepcopy(self.cloud_ga_qc)
                if self.cloud == 99:
                    self.cloud_frac = 999
                    self.cloud_frac_qc = '3'
                else:
                    self.cloud_frac = (100.0/8.0)*self.cloud
                    self.cloud_frac_qc = copy.deepcopy(self.cloud_ga_qc)

        #GD - 2 given coverage codes --> oktas/descriptive and descriptive (which can be converted to oktas)
        elif identifier in ('GD1', 'GD2', 'GD3', 'GD4', 'GD5', 'GD6'):
            #first summation layer
            if identifier == 'GD1':
                #oktas/descriptive coverage code 
                self.cloud_gd = int(block[1:3])
                self.cloud_gd_qc = block[3]
                #convert descriptive codes to oktas
                if self.cloud_gd == 11:
                    self.cloud_gd = 3
                elif self.cloud_gd == 12:
                    self.cloud_gd = 4
                elif self.cloud_gd == 13:
                    self.cloud_gd = 4
                elif self.cloud_gd == 14:
                    self.cloud_gd = 5
                elif self.cloud_gd == 15:
                    self.cloud_gd = 6
                elif self.cloud_gd == 16:
                    self.cloud_gd = 6
                elif self.cloud_gd == 17:
                    self.cloud_gd = 7
                elif self.cloud_gd == 18:
                    self.cloud_gd = 8
                elif self.cloud_gd == 19:
                    self.cloud_gd = 8
                #cannot create coverage code to oktas or missing?
                elif self.cloud_gd in [9,10,99]:
                    #check if descriptive code is valid
                    new_value = int(block[0])
                    #convert code to oktas
                    if new_value == 0:
                        self.cloud_gd = 0
                    elif new_value == 1:
                        self.cloud_gd = 2
                    elif new_value == 2:
                        self.cloud_gd = 4
                    elif new_value == 3:
                        self.cloud_gd = 6
                    elif new_value == 4:
                        self.cloud_gd = 8
                    #cannot create coverage code to oktas or missing?
                    else:
                        self.cloud_gd = 99
                        self.cloud_gd_qc = '3'

            #have higher coverage value in other layer? 
            elif identifier in ('GD2', 'GD3', 'GD4', 'GD5', 'GD6'):
                new_layer_coverage = int(block[1:3])
                new_layer_qc = block[3]
                #convert descriptive codes to oktas
                if new_layer_coverage == 11:
                    new_layer_coverage = 3
                elif new_layer_coverage == 12:
                    new_layer_coverage= 4
                elif new_layer_coverage == 13:
                    new_layer_coverage = 4
                elif new_layer_coverage == 14:
                    new_layer_coverage = 5
                elif new_layer_coverage == 15:
                    new_layer_coverage = 6
                elif new_layer_coverage == 16:
                    new_layer_coverage = 6
                elif new_layer_coverage == 17:
                    new_layer_coverage = 7
                elif new_layer_coverage == 18:
                    new_layer_coverage = 8
                elif new_layer_coverage == 19:
                    new_layer_coverage = 8
                #cannot create coverage code to oktas or missing?
                elif new_layer_coverage in [9,10,99]:
                    #check if descriptive code is valid
                    des_new_layer_coverage = int(block[0])
                    #convert code to oktas
                    if des_new_layer_coverage == 0:
                        new_layer_coverage = 0
                    elif des_new_layer_coverage == 1:
                        new_layer_coverage = 2
                    elif des_new_layer_coverage == 2:
                        new_layer_coverage = 4
                    elif des_new_layer_coverage == 3:
                        new_layer_coverage = 6
                    elif des_new_layer_coverage == 4:
                        new_layer_coverage = 8
                    #cannot create coverage code to oktas or missing?
                    else:
                        new_layer_coverage = 99
                        new_layer_qc = '3'

                #if current cloud_gd_qc is erroneous/suspect or data is missing, and this is not the case for the new data, then overwrite cloud_gd
                if (self.cloud_gd_qc in ['2','3','6','7']) or (self.cloud_gd == 99): 
                    if (new_layer_qc not in ['2','3','6','7']) & (new_layer_coverage != 99):
                        self.cloud_gd =    copy.deepcopy(new_layer_coverage)
                        self.cloud_gd_qc = copy.deepcopy(new_layer_qc)
                #else if both current cloud_gd_qc and new_qc are not erroneous/suspect and data is not missing, then check if new coverage is > cloud_gd
                elif (self.cloud_gd_qc not in ['2','3','6','7']) & (self.cloud_gd != 99) & (new_layer_qc not in ['2','3','6','7']) & (new_layer_coverage != 99):
                    if new_layer_coverage > self.cloud_gd:
                        self.cloud_gd = copy.deepcopy(new_layer_coverage)
                        self.cloud_gd_qc = copy.deepcopy(new_layer_qc)
                
                    
            #check if cloud is set, if not set it 
            if not self.have_cloud:
                self.cloud =    copy.deepcopy(self.cloud_gd)
                self.cloud_qc = copy.deepcopy(self.cloud_gd_qc)
                if self.cloud == 99:
                    self.cloud_frac = 999
                    self.cloud_frac_qc = '3'
                else:
                    self.cloud_frac = (100.0/8.0)*self.cloud
                    self.cloud_frac_qc = copy.deepcopy(self.cloud_gd_qc)
                self.have_cloud = True
            #else if cloud qc code is suspect/erroneous or data missing, and that is not the case with gd data, overwrite cloud 
            elif (self.cloud_qc in ['2','3','6','7']) or (self.cloud == 99):
                if self.cloud_gd_qc not in ['2','3','6','7'] & (self.cloud_gd != 99):
                    self.cloud =    copy.deepcopy(self.cloud_gd)
                    self.cloud_qc = copy.deepcopy(self.cloud_gd_qc)
                    if self.cloud == 99:
                        self.cloud_frac = 999
                        self.cloud_frac_qc = '3'
                    else:
                        self.cloud_frac = (100.0/8.0)*self.cloud
                        self.cloud_frac_qc = copy.deepcopy(self.cloud_gd_qc)

        elif identifier == 'SA1':
            if print_info:
                print(isd_map[identifier][map_value])
            # read part that contains sea surface temperature
            self.sst = (float(block[0:4]) / scale_factor) + kelvin
            self.sst_qc = block[4]
            if self.sst == 99.9:
                self.sst_qc = '3'

        elif identifier == 'ST1':
            if print_info:
                print(isd_map[identifier][map_value])
            # read part that contains soil temperature data
            self.soil_temp = (float(block[1:6]) / scale_factor) + kelvin
            self.soil_temp_qc = block[6]
            if self.soil_temp == 999.9:
                self.soil_temp_depth_qc = '3'
            self.soil_temp_depth = float(block[7:11]) / scale_factor
            self.soil_temp_depth_qc = block[11]
            if self.soil_temp_depth == 999.9:
                self.soil_temp_depth_qc = '3'

       # Read Weather Condition Codes (includes dust-related codes)
        elif identifier in ('AT1', 'AT2', 'AT3', 'AT4', 'AT5', 'AT6', 'AT7', 'AT8'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.at_weather_type{identifier[-1]}_wf = block[2:4]')
            exec(f'self.at_weather_ab_type{identifier[-1]}_wf = block[4:8]')

        elif identifier in ('AU1', 'AU2', 'AU3', 'AU4', 'AU5', 'AU6', 'AU7', 'AU8', 'AU9'):
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.au_proximity{identifier[-1]}_wf = block[0]')
            exec(f'self.au_descriptor{identifier[-1]}_wf = block[1]')
            exec(f'self.au_precip_code{identifier[-1]}_wf = block[2:4]')
            exec(f'self.au_obscuration{identifier[-1]}_wf = block[4]')
            exec(f'self.au_other_weather{identifier[-1]}_wf = block[5]')
            exec(f'self.au_combination{identifier[-1]}_wf = block[6]')
            if print_info:
                print(isd_map[identifier][map_value])

        elif identifier in ('AW1', 'AW2', 'AW3', 'AW4'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.aw_auto_atm_condition{identifier[-1]}_wf = block[0:2]')

        elif identifier in ('MV1', 'MV2', 'MV3', 'MV4', 'MV5', 'MV6', 'MV7'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (6,7):
                return
            exec(f'self.mv_atm_condition{identifier[-1]}_wf = block[0:2]')

        elif identifier in ('MW1', 'MW2', 'MW3', 'MW4', 'MW5', 'MW6', 'MW7'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.mw_man_atm_condition{identifier[-1]}_wf = block[0:2]')

        if print_block:
            print('read following block: {}'.format(block))

    def parse_rem_section(self, block):
        """Function to read the Remarks Data Section"""

        # initialize position, after REM and fist id
        info = self.section_chars * 2
        # read how many chars the REM has
        to_read = int(block[info:info + self.section_chars]) + self.section_chars * 3  # the ids we read already
        # start position
        pos = 0
        self.rem_content = block[pos:to_read]

        # # sanity check
        # print('block: {}'.format(block))
        # print('rem content: {}'.format(rem_content))
        return self.rem_content

    def parse_eqd_section(self, block):
        """Function to read the Element Quality Data Section"""

        param_code = 6  # length of the original value text & parameters' code
        rcode_length = 1  # reason code length
        pos = self.section_chars  # starting from 3, after reading section id
        read = self.section_chars

        while pos < len(block):
            # quality identifier
            qid = block[pos:pos + self.section_chars]
            # if qid == 'QNN':
            #     raise ValueError
            # update position
            pos += self.section_chars
            # original value text
            orig_text = block[pos:pos + param_code]
            # update position
            pos += param_code
            # reason code
            reason_code = block[pos:pos + rcode_length]
            # update position
            pos += rcode_length
            # parameter identifier
            parameter = block[pos:pos + param_code]
            # update position
            pos += param_code

            # characters read so far
            read += len(qid + orig_text + reason_code + parameter)

            if 'WOSPD' in parameter:
                self.speed_orig_value = orig_text
                self.speed_reason_code = reason_code

            if 'WODIR' in parameter:
                self.direction_orig_value = orig_text
                self.direction_reason_code = reason_code

            if 'ATOD' in parameter:
                self.dpoint_orig_value = orig_text
                self.dpoint_reason_code = reason_code

            if 'ATOT' in parameter:
                self.temp_orig_value = orig_text
                self.temp_reason_code = reason_code

            if 'APOSLP' in parameter:
                self.pressure_orig_value = orig_text
                self.pressure_reason_code = reason_code

            if 'SCOCIG' in parameter:
                self.ceiling_orig_value = orig_text
                self.ceiling_reason_code = reason_code

            if 'VODIS' in parameter:
                self.distance_orig_value = orig_text
                self.distance_reason_code = reason_code

        return read

    def parse_qnn_section(self):
        self.has_qnn = True
        # logging.info('You will not believe this! This line has a QNN section! Filename: '.format(self.filename[-17:0]))

class ParseUKAIRFile:
 
    '''class that handles processing of a UK AIR file'''
 
    def __init__(self,filename,uk_air_name):
 
        self.filename = filename
        self.parsed_data = {}
        self.parsed_data['start_dt'] = {}
        self.parsed_data['end_dt'] = {}
        self.parsed_data['measurements'] = {}
        self.parsed_data['units'] = {}
        self.parsed_data['flags'] = {}
        self.parsed_data['methods'] = {}
        self.uk_air_name = uk_air_name.lower()
 
    def read_data(self):

        read_active = False
        station_active = False

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')
            for row_count, row in enumerate(csvreader):
                
                # if have empty row, then continue
                if len(row) == 0:
                    continue

                # get station names
                elif 'status:' in row[0].lower():
                    read_active = False
                    station_active = True
                elif 'multi-day data' in row[0].lower():
                    read_active = False
                    station_column_names = ['',''] + row[1:]
                    stations = [col for col in station_column_names if col]
                elif station_active:
                    station_active = False
                    station_column_names = copy.deepcopy(row)
                    stations = [col for col in station_column_names if col]
                    
                # get column headers
                elif (row[0].lower() == 'date') or (row[0].lower() == 'start date'):
                    column_headers = [col.lower() for col in row]
                    
                    # reduce stations to only those which have parameter data
                    # at same time get data indices per station for parameter column
                    data_indices = {}
                    station_params = {}
                    for station_ii, station in enumerate(stations):
                        # cut column headers to those just relevant for station
                        station_start_column = station_column_names.index(station)
                        if station_ii == (len(stations)-1):
                            station_columns = column_headers[station_start_column:]
                        else:
                            station_end_column = station_column_names.index(stations[station_ii+1])
                            station_columns = column_headers[station_start_column:station_end_column]
                        # check in current parameter is available for station
                        if self.uk_air_name in station_columns:
                            data_indices[station] = station_start_column + station_columns.index(self.uk_air_name)
                            station_params[station] = self.uk_air_name
                    stations = list(data_indices.keys())
                    
                    # if no stations remain, then return from parsing file
                    if len(stations) == 0:
                        return

                    # setup lists for storing data per station
                    for station in stations: 
                        for parsed_data_var in self.parsed_data.keys():
                            if station not in self.parsed_data[parsed_data_var]:
                                self.parsed_data[parsed_data_var][station] = []
                    
                    # get indices for date
                    start_date_index = None
                    end_date_index = None
                    if 'date' in column_headers:
                        start_date_index = column_headers.index('date')
                    elif 'start date' in column_headers:
                        start_date_index = column_headers.index('start date')
                    if 'end date' in column_headers:
                        end_date_index = column_headers.index('end date')

                    # get index for time
                    time_index = None
                    if 'time' in column_headers:
                        time_index = column_headers.index('time')

                    # get number of time related columns
                    n_time_columns = 0
                    dt_indices = [start_date_index, end_date_index, time_index]
                    for dt_index in dt_indices:
                        if dt_index is not None:
                            n_time_columns += 1

                    # get index for units (per station)
                    merged_unit_flag = False
                    unit_indices = {}
                    if 'unit' in column_headers:
                        for station in stations:
                            cut_column_headers = column_headers[data_indices[station]:]
                            unit_indices[station] = cut_column_headers.index('unit') + data_indices[station]

                    # get index for flags (per station)
                    flag_indices = {}
                    if 'status' in column_headers:
                        for station in stations:
                            cut_column_headers = column_headers[data_indices[station]:]
                            flag_indices[station] = cut_column_headers.index('status') + data_indices[station]
                        # if have no unit indices then it is because it is merged with flags
                        if len(unit_indices) == 0:
                            merged_unit_flag = True
                            unit_indices = copy.deepcopy(flag_indices)

                    # get index for units and flags combined (per station)
                    elif 'status / units' in  column_headers:
                        merged_unit_flag = True
                        for station in stations:
                            cut_column_headers = column_headers[data_indices[station]:]
                            flag_indices[station] = cut_column_headers.index('status / units') + data_indices[station]
                        unit_indices = copy.deepcopy(flag_indices)

                    # set reading to be active
                    read_active = True

                # parse data line (per station)
                elif read_active:
                    
                    # if number of columns does not match the number of column headers then handle it
                    if len(column_headers) != len(row):

                        # if have 1 more column header than columns, then this is almost always due to missing units in last column 
                        # add empty units and continue 
                        if (len(column_headers) - len(row)) == 1:
                            row = row + ['']
                        # if have 1 more column than column headers, then this is almost always due to extra empty data column 
                        # cut off last column
                        elif (len(row) - len(column_headers)) == 1:
                            row = row[:len(column_headers)]
                        # in some cases the number of data columns is accidentially double and filled with empty data
                        # in these cases then cut off the empty half of data
                        elif ((len(column_headers) - n_time_columns) * 2) == len(row): 
                            row = row[:len(column_headers)]
                        # otherwise skip line
                        else:
                            #print('SKIP:{}vs{}'.format(len(column_headers), len(row)))
                            continue        

                    # process start/end dt
                    line_start_date = row[start_date_index]
                    if '-' in line_start_date:
                        start_date_split = line_start_date.split('-')
                    elif '/' in line_start_date:
                        start_date_split = line_start_date.split('/')
                    if len(start_date_split[0]) == 4:
                        start_date_year = int(start_date_split[0]) 
                        start_date_month = int(start_date_split[1])
                        start_date_day = int(start_date_split[2])
                    else:
                        start_date_year = int(start_date_split[2])
                        start_date_month = int(start_date_split[1])
                        start_date_day = int(start_date_split[0])

                    if end_date_index:
                        line_end_date = row[end_date_index]
                        if '-' in line_end_date:
                            end_date_split = line_end_date.split('-')
                        elif '/' in line_end_date:
                            end_date_split = line_end_date.split('/')
                        if len(end_date_split[0]) == 4:
                            end_date_year = int(end_date_split[0]) 
                            end_date_month = int(end_date_split[1])
                            end_date_day = int(end_date_split[2])
                        else:
                            end_date_year = int(end_date_split[2])
                            end_date_month = int(end_date_split[1])
                            end_date_day = int(end_date_split[0])

                    if time_index:
                        line_time = row[time_index]
                        time_split = line_time.split(':')
                        time_hour = int(time_split[0])
                        time_min = int(time_split[1])

                    if (start_date_index != None) & (end_date_index != None):
                        start_dt = datetime.datetime(year=start_date_year, month=start_date_month, day=start_date_day, hour=0, minute=0)
                        end_dt = datetime.datetime(year=end_date_year, month=end_date_month, day=end_date_day, hour=0, minute=0)
                    elif (start_date_index != None) & (time_index != None):
                        # in some cases reported end time will be within 00:00 and 01:00
                        # in these cases need to remove hour with relativedelta to handle wrapping of days/months/years
                        time_hour = time_hour - 1
                        remove_hour = False
                        if time_hour < 0:
                            remove_hour = True
                            time_hour = 0
                        start_dt = datetime.datetime(year=start_date_year, month=start_date_month, day=start_date_day, hour=time_hour, minute=time_min)
                        if remove_hour:
                            start_dt = start_dt - relativedelta(hours=1)
                        # handle 2 hour vs 1 hour resolution data
                        if '2H.csv' in self.filename:
                            delta_hours = 2
                            start_dt = start_dt - relativedelta(hours=1)
                        else:
                            delta_hours = 1

                        end_dt = start_dt + relativedelta(hours=delta_hours)
                    elif (start_date_index != None):
                        start_dt = datetime.datetime(year=start_date_year, month=start_date_month, day=start_date_day, hour=0, minute=0)
                        end_dt = start_dt + relativedelta(days=1)

                    # iterate through stations
                    for station in stations:

                        # append start/end dt
                        self.parsed_data['start_dt'][station].append(start_dt)
                        self.parsed_data['end_dt'][station].append(end_dt)

                        # parse units
                        if len(unit_indices) > 0:
                            station_unit = row[unit_indices[station]] 
                            if merged_unit_flag:
                                if station_unit.strip() == '':
                                    unit = ''
                                else:
                                    unit = station_unit.split(' ')[1]
                            else:
                                unit = station_unit
                            if unit.strip().lower() == 'no data':
                                unit = ''
                            self.parsed_data['units'][station].append(unit)
                        else:
                            self.parsed_data['units'][station].append('')
                                
                        # parse flags
                        if len(flag_indices) > 0:
                            station_flag = row[flag_indices[station]]
                            if merged_unit_flag:
                                if station_flag.strip() == '':
                                    flag = ''
                                else:
                                    flag = station_flag.split(' ')[0]
                            else:
                                flag = station_flag
                            if flag.strip().lower() == 'no data':
                                flag = ''
                            if flag != '':
                                self.parsed_data['flags'][station].append(flag)
                            else:
                                self.parsed_data['flags'][station].append('')
                        else:
                            self.parsed_data['flags'][station].append('')

                        # parse data
                        measurement = row[data_indices[station]]
                        if measurement.strip().lower() == 'no data':
                            measurement = ''
                        #less than (<) or greater than (>) signs are used in file to flag measurements below or above
                        # detection limits, add flags for this
                        if '<' in measurement:       
                            measurement = measurement[1:] 
                            if self.parsed_data['flags'][station][-1] == '':
                                self.parsed_data['flags'][station][-1] = '<'
                            else:
                                self.parsed_data['flags'][station][-1] += '|<'
                        elif '>' in measurement:
                            measurement = measurement[1:]  
                            if self.parsed_data['flags'][station][-1] == '':
                                self.parsed_data['flags'][station][-1] = '>'
                            else:  
                                self.parsed_data['flags'][station][-1] += '|>'       
                        self.parsed_data['measurements'][station].append(measurement)

                        # parse method (if in units)
                        if '(' in self.parsed_data['units'][station][-1]:
                            unit_split = self.parsed_data['units'][station][-1].split(' ')
                            self.parsed_data['units'][station][-1] = unit_split[0]
                            self.parsed_data['methods'][station].append(' '.join(unit_split[1:])[1:-1])
                        # handle some special cases
                        elif 'NH3_active.csv' in self.filename:
                            self.parsed_data['methods'][station].append('nh3active')
                        elif 'NH3_diffusiontube.csv' in self.filename:
                            self.parsed_data['methods'][station].append('nh3diffusiontube')
                        elif 'NH3_passive.csv' in self.filename:
                            self.parsed_data['methods'][station].append('nh3passive')
                        elif ('reflectance' in station_params[station]) or ('-TOR-' in station_params[station]):
                            self.parsed_data['methods'][station].append('improve-a')
                        elif ('transmittance' in station_params[station]) or ('-TOT-' in station_params[station]):
                            self.parsed_data['methods'][station].append('niosh-5040')
                        else:
                            self.parsed_data['methods'][station].append('')

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    for station in self.parsed_data[column_header].keys():
                        self.parsed_data[column_header][station] = np.array(self.parsed_data[column_header][station])
                else:
                    for station in self.parsed_data[column_header].keys():
                        self.parsed_data[column_header][station] = np.array(self.parsed_data[column_header][station], dtype=np.str)             


class ParseUSEPAAirNowDOS:

    '''class that handles processing of a US EPA AirNow DOS file'''

    def __init__(self,filename,run_details):
        self.filename = filename
        self.run_details = run_details

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    column_headers_to_parse = {'start_dt':-1,'measurement':column_headers.index('Raw Conc.'),'unit':column_headers.index('Conc. Unit'),'qa':column_headers.index('QC Name')}
                    column_headers_to_process = {'year':column_headers.index('Year'), 'month':column_headers.index('Month'), 'day':column_headers.index('Day'), 'hour':column_headers.index('Hour'), 'duration':column_headers.index('Duration')}
                    self.parsed_data = {column_header:[] for column_header in column_headers_to_parse}
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    for column_header in column_headers_to_parse:
                        if column_header == 'start_dt':
                            year = int(row[column_headers_to_process['year']])
                            month = int(row[column_headers_to_process['month']])
                            day = int(row[column_headers_to_process['day']])
                            hour = int(row[column_headers_to_process['hour']])
                            duration = row[column_headers_to_process['duration']]

                            if duration != '1 Hr':
                                self.run_details['log_file_str'] += 'NOT HOURLY RESOLUTION IN FILE: {} .STOPPING PROCESSING.'.format(self.filename)
                                create_GHOST_stage_logfile(2,self.run_details)

                            self.parsed_data['start_dt'].append(datetime.datetime(year=year,month=month,day=day,hour=hour,minute=0)) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                        else:
                            self.parsed_data[column_header].append(row[column_headers_to_parse[column_header]]) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 


class ParseUSEPAAQSFile:

    '''class that handles processing of a US EPA AQS file'''

    def __init__(self,filename,parameter_codes):

        self.filename = filename
        self.parameter_codes = parameter_codes
        #get resolution of file
        self.file_resolution = self.filename.split('/')[-1].split('_')[0]
        #get number of lines in file (for memory allocation)
        with open(self.filename) as f:
            n_lines = sum(1 for line in f) - 1
        #set column headers per file resolution
        if self.file_resolution == 'hourly':
            self.column_headers = ['station_reference','start_datetime','measurement','Parameter Code','POC','Latitude','Longitude','Datum','Units of Measure','MDL','Uncertainty','Qualifier','Method Code']
        elif self.file_resolution == 'daily':
            self.column_headers = ['station_reference','start_datetime','measurement','Parameter Code','POC','Latitude','Longitude','Datum','Units of Measure','Observation Percent','Method Code']
        #set empty dataframe to store parsed data
        standard_dtype = [(key,object) for key in self.column_headers]
        self.parsed_data = np.empty(n_lines, dtype=standard_dtype)

    def read_data(self):

        parsed_lines = 0
        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):
                
                #get indices associated with fields that need manipulating from first line
                if row_count == 0:
                   state_code_index = row.index('State Code') 
                   county_code_index = row.index('County Code') 
                   site_num_index = row.index('Site Num')
                   if self.file_resolution == 'hourly':
                       date_index = row.index('Date GMT')
                       time_index = row.index('Time GMT')
                       measurement_index = row.index('Sample Measurement')
                   elif self.file_resolution == 'daily':
                       date_index = row.index('Date Local')
                       measurement_index = row.index('Arithmetic Mean')
                   original_column_headers = copy.deepcopy(row)
                   #get indices for fields which do not need to manipulate
                   column_indices = {}
                   for header_ii, original_column_header in enumerate(original_column_headers):
                       if original_column_header in self.column_headers:
                           column_indices[original_column_header] = header_ii
                   
                #otherwise parse line    
                else:
                    #only parse line if parameter code is for current species
                    if row[column_indices['Parameter Code']] in self.parameter_codes:
                        
                        #generate station reference (State Code + County Code + Site Num)
                        self.parsed_data['station_reference'][parsed_lines] = row[state_code_index] + row[county_code_index] + row[site_num_index]

                        #parse datetime
                        split_date = row[date_index].split('-')
                        if self.file_resolution == 'hourly':
                            split_time = row[time_index].split(':')
                            self.parsed_data['start_datetime'][parsed_lines] = datetime.datetime(year=int(split_date[0]), month=int(split_date[1]), day=int(split_date[2]), hour=int(split_time[0]), minute=int(split_time[1]))
                        elif self.file_resolution == 'daily':
                            self.parsed_data['start_datetime'][parsed_lines] = datetime.datetime(year=int(split_date[0]), month=int(split_date[1]), day=int(split_date[2]), hour=0, minute=0)                       

                        #parse measurements
                        self.parsed_data['measurement'][parsed_lines] = row[measurement_index]
                        
                        #parse rest of fields
                        for column_header in column_indices.keys():
                            self.parsed_data[column_header][parsed_lines] = row[column_indices[column_header]]

                        #add to parsed line counter
                        parsed_lines += 1

        #delete empty lines
        if parsed_lines < len(self.parsed_data):
            self.parsed_data = self.parsed_data[:parsed_lines]
                

class ParseUSEPACASTNETFile:

    '''class that handles processing of a US EPA CASTNET file'''

    def __init__(self,filename):

        self.filename = filename

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 
           

class ParseUSNADPAMNetFile:

    '''class that handles processing of a US NADP AMNet file'''

    def __init__(self,filename):

        self.filename = filename

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 

            #convert missing measurements to NaN 
            self.parsed_data['PBM'] = self.parsed_data['PBM'].astype(np.float32)
            missing_inds = np.where(self.parsed_data['PBM'] == -9.0)[0]
            self.parsed_data['PBM'][missing_inds] = np.NaN 

            
class ParseUSNADPAMoNFile:

    '''class that handles processing of a US NADP AMoN file'''

    def __init__(self,filename):

        self.filename = filename

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 

            #convert missing measurements to NaN 
            self.parsed_data['CONC'] = self.parsed_data['CONC'].astype(np.float32)
            missing_inds = np.where(self.parsed_data['CONC'] == -9.0)[0]
            self.parsed_data['CONC'][missing_inds] = np.NaN 


class ParseWMOWDCGGFile:

    '''class that handles processing of a WMO WDCGG file'''

    def __init__(self,filename):

        self.filename = filename
        self.read_header_global = False
        self.read_header_variable = False
        self.read_header_global_start = -999
        self.read_header_variable_start = -999

        self.global_attributes = {}
        self.variable_attributes = {} 
        self.parsed_data = {}
         

    def read_metadata(self):

        #read header
        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=':', quoting=csv.QUOTE_NONE)
            for row_count, row in enumerate(csvreader):

                if row_count == 0:
                    self.n_header_lines = int(row[-1].strip())
                elif row_count == self.read_header_global_start:
                    self.read_header_global = True
                    self.read_header_variable = False
                elif row_count == self.read_header_variable_start:
                    self.read_header_global = False
                    self.read_header_variable = True
                elif row_count == (self.n_header_lines-1):
                    self.read_header_global = False
                    self.read_header_variable = False
                    break

                if '# GLOBAL ATTRIBUTES' in row:
                    self.read_header_global_start = row_count + 2
        
                elif '# VARIABLE ATTRIBUTES' in row:
                    self.read_header_variable_start = row_count + 2
               
                #parse global header attributes
                if self.read_header_global:
                    if len(row) >= 2:
                        if row[1].strip() == 'comment':
                            global_attribute_name = '{}:{}'.format(row[0][1:].strip(), row[1].strip())
                            if len(row) > 3:
                                global_attribute_value = ':'.join([row_value.strip() for row_value in row[2:]])
                            else:
                                global_attribute_value = row[2].strip()
                        else:
                            global_attribute_name = row[0][1:].strip()
                            if len(row) > 2:
                                global_attribute_value = ':'.join([row_value.strip() for row_value in row[1:]])
                            else:
                                global_attribute_value = row[1].strip()
                        #assign value
                        if global_attribute_name in list(self.global_attributes.keys()):
                            self.global_attributes[global_attribute_name] = '{};{}'.format(self.global_attributes[global_attribute_name], global_attribute_value)
                        else:
                            self.global_attributes[global_attribute_name] = copy.deepcopy(global_attribute_value)

                #parse variable header attributes
                elif self.read_header_variable:
                    if len(row) >= 3:
                        variable_attribute_name = '{}:{}'.format(row[0][1:].strip(), row[1].strip())
                        if len(row) > 3:
                            variable_attribute_value = ':'.join([row_value.strip() for row_value in row[2:]])
                        else:
                            variable_attribute_value = row[2].strip()
                        #assign value
                        if variable_attribute_name in list(self.variable_attributes.keys()):
                            self.variable_attributes[variable_attribute_name] = '{};{}'.format(self.variable_attributes[variable_attribute_name], variable_attribute_value)
                        else:
                            self.variable_attributes[variable_attribute_name] = copy.deepcopy(variable_attribute_value)

    def read_data(self):

        #read data
        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=' ', quoting=csv.QUOTE_NONE)
            for row_count, row in enumerate(csvreader):
            
                if row_count == (self.n_header_lines-1):
                    column_headers_orig = ['site_gaw_id','start_year','start_month','start_day','start_hour','start_minute','start_second','end_year','end_month','end_day','end_hour','end_minute','end_second','value','value_unc','nvalue','latitude','longitude','altitude','elevation','intake_height','flask_no','ORG_QCflag','QCflag','instrument','measurement_method','scale']
                    column_headers_new = ['start_dt','end_dt','value','value_unc','latitude','longitude','elevation','intake_height','QCflag','instrument','measurement_method','scale']
                    for column_header in column_headers_new:
                        self.parsed_data[column_header] = []

                    #get indices for dates
                    start_year_index = column_headers_orig.index('start_year')
                    start_month_index = column_headers_orig.index('start_month')
                    start_day_index = column_headers_orig.index('start_day')      
                    start_hour_index = column_headers_orig.index('start_hour')
                    start_minute_index = column_headers_orig.index('start_minute')
                    start_second_index = column_headers_orig.index('start_second')

                    end_year_index = column_headers_orig.index('end_year')
                    end_month_index = column_headers_orig.index('end_month')
                    end_day_index = column_headers_orig.index('end_day')
                    end_hour_index = column_headers_orig.index('end_hour')
                    end_minute_index = column_headers_orig.index('end_minute')
                    end_second_index = column_headers_orig.index('end_second')

                elif row_count >= self.n_header_lines:
                    #parse start_dt
                    start_year = int(row[start_year_index])
                    start_month = int(row[start_month_index])
                    start_day = int(row[start_day_index])
                    start_hour = int(row[start_hour_index])
                    start_minute = int(row[start_minute_index])
                    start_second = int(row[start_second_index])
                    if (start_year >= 0) & (start_month >= 0) & (start_day >= 0) & (start_hour >= 0) & (start_minute >= 0) & (start_second >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=start_hour,minute=start_minute,second=start_second)
                    elif (start_year >= 0) & (start_month >= 0) & (start_day >= 0) & (start_hour >= 0) & (start_minute >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=start_hour,minute=start_minute)
                    elif (start_year >= 0) & (start_month >= 0) & (start_day >= 0) & (start_hour >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=start_hour,minute=0)
                    elif ('_daily' in self.filename) & (start_year >= 0) & (start_month >= 0) & (start_day >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=0,minute=0)
                    elif ('_monthly' in self.filename) & (start_year >= 0) & (start_month >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=1,hour=0,minute=0)
                    #continue to next row as cannot parse start time
                    else:
                        continue                         

                    #parse end dt (if not given, make from stated sampling frequency/file resolution - if event, assume sampling is 1 minute)
                    end_year = int(row[end_year_index])
                    end_month = int(row[end_month_index])
                    end_day = int(row[end_day_index])
                    end_hour = int(row[end_hour_index])
                    end_minute = int(row[end_minute_index])
                    end_second = int(row[end_second_index])
                    if (end_year >= 0) & (end_month >= 0) & (end_day >= 0) & (end_hour >= 0) & (end_minute >= 0) & (end_second >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=end_hour,minute=end_minute,second=end_second)
                    elif (end_year >= 0) & (end_month >= 0) & (end_day >= 0) & (end_hour >= 0) & (end_minute >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=end_hour,minute=end_minute)
                    elif (end_year >= 0) & (end_month >= 0) & (end_day >= 0) & (end_hour >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=end_hour,minute=0)
                    elif ('_daily' in self.filename) & (end_year >= 0) & (end_month >= 0) & (end_day >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=0,minute=0)
                    elif ('_monthly' in self.filename) & (end_year >= 0) & (end_month >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=1,hour=0,minute=0)
                    #if not end time provided, generate from file resolution
                    else:
                        if '_hourly' in self.filename:
                            e_dt = s_dt + relativedelta(hours=1)
                        elif '_daily' in self.filename:
                            e_dt = s_dt + relativedelta(days=1)
                        elif '_monthly' in self.filename:
                            e_dt = s_dt + relativedelta(months=1)
                        elif '_event' in self.filename:
                            e_dt = s_dt + relativedelta(minutes=1)
                            
                    #append start_dt/end_dt         
                    self.parsed_data['start_dt'].append(s_dt)
                    self.parsed_data['end_dt'].append(e_dt)

                    #now append rest of row 
                    for column_header, row_value in zip(column_headers_orig, row):
                        if column_header not in ['site_gaw_id','nvalue','altitude','flask_no','ORG_QCflag','start_year','start_month','start_day','start_hour','start_minute','start_second','end_year','end_month','end_day','end_hour','end_minute','end_second']:
                            self.parsed_data[column_header].append(row_value) 


        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str) 
                
                #set fill values as empty strings
                nan_inds = np.where(self.parsed_data[column_header] == self.variable_attributes['{}:_FillValue'.format(column_header)])[0]                        
                self.parsed_data[column_header][nan_inds] = ''