from calendar import monthrange
import copy
import csv
import datetime
from dateutil.relativedelta import relativedelta
from html.parser import HTMLParser
from xml.etree import ElementTree as ET
from io import StringIO
import logging
import struct

import numpy as np
import pandas as pd
import json

from crossnetwork_processing_functions import create_GHOST_stage_logfile, isUTF8
from GHOST_standards import standard_parameters


class ParseAERONETFile(HTMLParser):

    '''class that handles parsing of an AERONET html file'''

    def __init__(self):
        super(ParseAERONETFile, self).__init__(convert_charrefs=True)

        self.have_title = False
        self.have_contact = False
        self.have_column_names = False

    def handle_starttag(self, tag, attrs):
        if tag == 'title':
            self.have_title = True

    def handle_endtag(self, tag):
        pass

    def handle_data(self, data):
        if (self.have_title) & (self.have_contact) & (self.have_column_names):
            data_split = data.strip("\r\n").split(',')
            self.current_n_columns = len(data_split)
            #if number of current line columns is < than number of data columns, do not process line
            if self.current_n_columns >= self.n_columns:
                #if number of current line columns is != to previous number of accepted columns, do not process line
                if (self.previous_n_columns == 0) or (self.previous_n_columns == self.current_n_columns):
                    for column_ii, column_name in enumerate(self.column_names):
                        column_val = data_split[column_ii]
                        if column_name not in ['AERONET_Site','AERONET_Site_Name','Date(dd:mm:yyyy)','Date_(dd:mm:yyyy)','Time(hh:mm:ss)','Time_(hh:mm:ss)','Day_of_Year','Day_of_Year(Fraction)','Last_Processing_Date(dd:mm:yyyy)','Last_Processing_Date','Last_Date_Processed','Last_Processing_Time(hh:mm:ss)','Inversion_Data_Quality_Level','Data_Quality_Level','Retrieval_Measurement_Scan_Type','Measurement_Type(solarorlunar)']:
                            #convert missing measurements to nan
                            column_val = float(column_val)
                            if column_val == -999:
                                column_val = np.NaN
                        self.var_data[column_name].append(column_val)
                    self.previous_n_columns = self.current_n_columns
                    

        elif (self.have_title) & (self.have_contact):
            self.column_names = data.strip("\r\n").replace(" ", "").split(',')
            self.n_columns = len(self.column_names)
            self.previous_n_columns = 0
            self.var_data = {k:[] for k in self.column_names}
            self.have_column_names = True

        elif self.have_title:
            if 'Contact:' in data:
                contact_str = data.strip("\r\n").replace(" ", "")
                #get contact name
                if "PI=" in contact_str:
                    self.contact_name = contact_str.split("PI=")[1].split(";")[0].replace("_", " ")
                #default contact name
                else:
                    self.contact_name = 'Brent Holben'
                #get contact email address
                if "PIEmail=" in contact_str:
                    self.contact_email_address = contact_str.split("PIEmail=")[1].replace("_", " ")
                #default contact email address
                else:
                    self.contact_email_address = 'Brent.N.Holben@nasa.gov'
                #contact institution
                self.contact_institution = 'NASA'
                self.have_contact = True

                #handle case where contact line is not line broken and the column names directly follow
                if "AERONET_Site," in data:
                    column_name_line = data.strip("\r\n").replace(" ", "")
                    column_name_line = 'AERONET_Site,Date'+column_name_line.split('AERONET_Site,Date')[-1]
                    self.column_names = column_name_line.split(',')
                    self.n_columns = len(self.column_names)
                    self.previous_n_columns = 0
                    self.var_data = {k:[] for k in self.column_names}
                    self.have_column_names = True


class ParseBJMEMCFile:

    '''class that handles processing of a BJMEMC file'''

    def __init__(self,filename,run_details):

        self.filename = filename
        self.run_details = run_details
        self.parsed_data = {}

    def read_data(self):

        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')

            for row_count, row in enumerate(csvreader):

                #column headers
                # need to get unique column headers as can have a few station column headers which are repeated 
                # in these cases just keep first instance
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    unique_column_headers = np.unique(column_headers)
                    header_indices = {column_header:column_headers.index(column_header) for column_header in unique_column_headers} 
                    for column_header in unique_column_headers:
                        if column_header not in ['date','hour','type']:
                            self.parsed_data[column_header] = []
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    #make sure number of columns on line is same as number of column headers, if not then skip line
                    if len(row) == len(column_headers): 
                        param = row[header_indices['type']]
                        #only parse data for current parameter
                        if param == self.run_details['parameter']:
                            #parse date
                            date = row[header_indices['date']]
                            year = int(date[:4])
                            month = int(date[4:6])
                            day = int(date[6:])
                            hour = int(row[header_indices['hour']])
                            self.parsed_data['start_dt'].append(datetime.datetime(year=year,month=month,day=day,hour=hour,minute=0)) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                            #parse station measurements
                            for column_header in unique_column_headers:
                                if column_header not in ['date','hour','type']:
                                    self.parsed_data[column_header].append(row[header_indices[column_header]])
                            
            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 


class ParseCANADANAPSFile:

    '''class that handles processing of a CANADA NAPS file'''

    def __init__(self,filename,run_details,NAPS_code,all_NAPS_codes,data_year, parameter_details):

        self.filename = filename
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.parameter = copy.deepcopy(run_details['parameter'])
        self.NAPS_code = NAPS_code
        self.all_NAPS_codes = all_NAPS_codes
        self.data_year = str(data_year)

        #determine file type
        #1. csv?
        if self.filename[-4:] == '.csv':
            self.file_type = 'csv'
        #2. XLS
        elif (self.filename[-4:] == '.XLS') or (self.filename[-4:] == '.xls'):
            self.file_type = 'xls'
            self.data_sheet_names = ['Data']
        #3. xlsx
        elif (self.filename[-5:] == '.xlsx') or (self.filename[-5:] == '.XLSX'):
            self.file_type = 'xlsx' 
            if self.parameter == 'HCHO':
                self.data_sheet_names = ['Carbonyls']
            elif self.parameter in ['CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE']:
                self.data_sheet_names = ['VOC']
            elif self.parameter == 'PM10':
                self.data_sheet_names = ['PM2.5','PM2.5-10']
            elif self.parameter == 'PM2.5':
                self.data_sheet_names = ['PM2.5']
            elif self.parameter in ['PM10_Ca++','PM10_Cl-','PM10_K+','PM10_Mg++','PM10_MSA','PM10_Na+','PM10_NH4+','PM10_NO3-','PM10_SO4--','PM2.5_Ca++','PM2.5_Cl-','PM2.5_K+','PM2.5_Mg++','PM2.5_MSA','PM2.5_Na+','PM2.5_NH4+','PM2.5_NO3-','PM2.5_SO4--']:
                self.data_sheet_names = ['Ions-Spec_IC']
            elif self.parameter in ['PM10_Al','PM10_As','PM10_Cd','PM10_Co','PM10_Cr','PM10_Cu','PM10_Fe','PM10_Mn','PM10_Ni','PM10_Pb','PM10_Se','PM10_V','PM10_Zn','PM2.5_Al','PM2.5_As','PM2.5_Cd','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_Fe','PM2.5_Mn','PM2.5_Ni','PM2.5_Pb','PM2.5_Se','PM2.5_V','PM2.5_Zn']:
                self.data_sheet_names = ['Metals_ICPMS (Near-Total)','Elements_EXDXRF','Elements_EDXRF']
            elif self.parameter in ['PM10_C','PM10_EC','PM10_OC','PM2.5_C','PM2.5_EC','PM2.5_OC']:
                self.data_sheet_names = ['OCEC & OC Artifacts']
            elif self.parameter in ['NH3']:
                self.data_sheet_names = ['Ammonia_IC']
            elif self.parameter in ['HNO3','SO2']:
                self.data_sheet_names = ['Acidic Gaseous PM Precursors_IC']
        self.all_parsed_metadata = []
        self.all_parsed_data = []

        #set dict for helping parse some badly formatted dates
        self.month_dict = {'JAN':'01','FEB':'02','MAR':'03','APR':'04','MAY':'05','JUN':'06','JUL':'07','AUG':'08','SEP':'09','OCT':'10','NOV':'11','DEC':'12'}

        # set flag for precip metals
        self.metals = False
        if self.run_details['parameter'] in ['Al_PREC', 'As_PREC', 'Cd_PREC', 'Co_PREC', 'Cr_PREC', 'Cu_PREC', 'Fe_PREC', 'Mn_PREC', 'Ni_PREC', 'Pb_PREC', 'Se_PREC', 'V_PREC', 'Zn_PREC']:
            self.metals = True

    def read_data(self):
       
        #read csv file type
        if self.file_type == 'csv':

            if 'PREC' in self.parameter: # files of precip similar to CAPMoN files
                parsed_data = {'station_reference':[], 'start_dt':[], 'end_dt':[], 'measurement':[], 'flags':[]}
                parsed_metadata = {}

                if self.parameter == 'PREC_ACCUM':
                    #self.lod_ref = 'StandardGaugeDepth_mm_DetL'
                    self.flag_ref = 'StandardGaugeDepth_mm_Flag'
                    self.meas_ref = 'StandardGaugeDepth_mm'
                elif '_PREC' in self.parameter:
                    if self.metals:
                        #self.lod_ref = self.parameter_details['chemical_formula']+'_mgL_DetL'
                        if self.run_details['parameter'] == 'Zn_PREC':
                            self.flag_ref = self.parameter_details['chemical_formula']+'_NGL_Flag'
                            self.meas_ref = self.parameter_details['chemical_formula']+'_NGL'
                        else:
                            self.flag_ref = self.parameter_details['chemical_formula'].upper()+'_NGL_Flag'
                            self.meas_ref = self.parameter_details['chemical_formula'].upper()+'_NGL'
                    else:
                        #self.lod_ref = self.parameter_details['chemical_formula']+'_mgL_DetL'
                        self.flag_ref = self.parameter_details['chemical_formula']+'_mgL_Flag'
                        self.meas_ref = self.parameter_details['chemical_formula']+'_mgL'
                 
                active_meta = False
                active_table = False
                table_name = ''
                
                with open(self.filename, newline='', encoding='latin1') as csvfile:
                    csvreader = csv.reader(csvfile, delimiter=',', quotechar='"', skipinitialspace=True)
                    for row_count, row in enumerate(csvreader):
                    
                        #empty row
                        if len(row) == 0:
                            continue

                        #remove french translation from row columns
                        row = [r.split('//')[0].strip() for r in row]

                        #file revision date?
                        if row[0] == '*DATE THIS FILE GENERATED':
                            parsed_metadata['file_revision_date'] = datetime.datetime.strptime(row[1], '%Y-%m-%d')

                        # get analytical method per component
                        if row[0] == '*TABLE COLUMN LABORATORY ANALYTICAL METHOD':
                            analytical_methods = row[1:]

                        #beginning of data table?
                        if row[0] == '*TABLE NAME':
                            #site information
                            if row[1] == 'Site information':
                                table_name = 'site_information'
                            #instrument detection limits
                            elif row[1] == 'Instrument Detection Limits':
                                table_name = 'instrument_detection_limits'
                            #data
                            elif row[1] in ['CAPMoN_Ozone','GLBM METALS', 'CAPMoNPrecipitationChemistry', 'CAPMoN_Precip_Chemistry', 'AlbertaPrecip']:
                                table_name = 'data'
                            else:
                                table_name = ''

                            if table_name != '':
                                active_meta = True
                            continue

                        #start of data table?
                        if (row[0] == '*TABLE DATA BEGINS') & (table_name != ''):
                            active_meta = False
                            active_table = True
                            continue

                        #end of data table?
                        if row[0] == '*TABLE DATA ENDS':
                            active_table = False

                        #active table meta?
                        if active_meta:
                            #fill codes
                            if row[0] == '*TABLE COLUMN MISSING CODE':
                                missing_codes = row[1:]
                                #transform '-999.' to '-999.0' to correctly catch missing codes
                                missing_codes = ['-999.0' if missing_code == '-999.' else missing_code for missing_code in missing_codes]

                            #column names
                            if row[0] == '*TABLE COLUMN NAME--SHORT FORM': 
                                column_names_raw = row[1:]
                                column_names = []
                                for column_name_raw in column_names_raw: # do correction to standardise column names for all files
                                    column_names.append(column_name_raw.replace('__', '_').replace('flag', 'Flag').replace('h_', 'H_').replace('DeptH', 'Depth'))

                                if table_name == 'site_information':
                                    for column_name in column_names:
                                        if column_name != '':
                                            if column_name == 'SiteID':
                                                parsed_metadata['station_reference'] = []
                                            elif column_name == 'SiteLandUse':
                                                parsed_metadata['Land_Use'] = []
                                            else:
                                                parsed_metadata[column_name.lower()] = []    

                        #parse data line
                        if active_table:
                            row = row[1:]

                            # check if header and data have same length
                            if len(row) != len(missing_codes):
                                n_diff_columns = abs(len(row) - len(missing_codes))
                                missing_codes = missing_codes[:-n_diff_columns] # cut last column of header

                            #replace fill values with empty strings
                            for column_ii, missing_code in enumerate(missing_codes):
                                if row[column_ii] == missing_code:
                                    row[column_ii] = ''

                            # check if header and data have same length
                            if len(row) != len(column_names):
                                n_diff_columns = abs(len(row) - len(column_names))
                                column_names = column_names[:-n_diff_columns] # cut last column of header

                            #metadata
                            if table_name == 'site_information':
                                #print(column_names)
                                for column_name in column_names:
                                    if column_name != '':
                                        if column_name == 'SiteName':
                                            parsed_metadata[column_name.lower()].append(row[column_names.index(column_name)].split(';')[0])
                                        elif column_name == 'SiteID':
                                            parsed_metadata['station_reference'].append(row[column_names.index(column_name)])
                                        elif column_name == 'SiteLandUse':
                                            parsed_metadata['Land_Use'].append(row[column_names.index(column_name)])
                                        else:
                                            parsed_metadata[column_name.lower()].append(row[column_names.index(column_name)])
                                            
                            elif table_name == 'instrument_detection_limits':
                                #siteid
                                self.parsed_detection_limits['station_reference'].append(row[column_names.index('SiteID')])
                                #year
                                self.parsed_detection_limits['year'].append(row[column_names.index('Year')])
                                #detection limit
                                self.parsed_detection_limits['instrument_detection_limit'].append(row[column_names.index(self.lod_ref)])
                                
                                
                            #measurement data
                            elif table_name == 'data':
                                print('column names ', column_names)
                                # Analytical instrument: only parse the one for parameter
                                parsed_metadata['analytical instrument'] = analytical_methods[column_names.index(self.meas_ref)]

                                #SiteID 
                                parsed_data['station_reference'].append(row[column_names.index('SiteID')])

                                #parse datetime
                                start_date = row[column_names.index('DateStartUTC')].split('-')
                                start_time = row[column_names.index('TimeStartUTC')].split(':')
                                end_date = row[column_names.index('DateEndUTC')].split('-')
                                end_time = row[column_names.index('TimeEndUTC')].split(':')
                                parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[0]),month=int(start_date[1]),day=int(start_date[2]),hour=int(start_time[0]),minute=int(start_time[1])))
                                parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[0]),month=int(end_date[1]),day=int(end_date[2]),hour=int(end_time[0]),minute=int(end_time[1])))

                                # parse measurements, flags
                                parsed_data['measurement'].append(row[column_names.index(self.meas_ref)])
                                parsed_data['flags'].append(row[column_names.index(self.flag_ref)])
                                # if '_PREC' in self.run_details['parameter']:
                                #     parsed_data['lods'].append(row[column_names.index(self.lod_ref)])
                                # else:
                                #     pass
                
                    #add to all parsed lists
                    self.all_parsed_data.append(parsed_data)
                    self.all_parsed_metadata.append(parsed_metadata)    

            else: # all non precip files
                parsed_metadata = {}
                parsed_data = {'start_dt':[],'end_dt':[],'measurement':[],'station_reference':[], 'longitude':[], 'latitude':[], 'station_name':[]}

                start_read = False

                #get encoding of data file
                is_utf8 = isUTF8(self.filename)
                if is_utf8:
                    file_encoding = 'UTF-8-SIG'
                else:
                    file_encoding = 'ISO-8859-1'

                with open(self.filename, newline='',encoding=file_encoding) as csvfile:
                    csvreader = csv.reader(csvfile, delimiter=',', quotechar='"', skipinitialspace=True)
                    for row_count, row in enumerate(csvreader):

                        #empty row then continue
                        if len(row) == 0:
                            continue

                        #remove french from header information 
                        if start_read == False:
                            row = [col.split('//')[0].replace(' ','') for col in row]

                        #get file revision date
                        if 'filegeneratedon' in row[0].lower():  
                            revision_date = ','.join(row).lower().replace('filegeneratedon','').replace(',','').replace(':','').strip()
                            if '-' in revision_date:
                                rev_date_split = revision_date.split('-')
                                parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[0]), month=int(rev_date_split[1]), day=int(rev_date_split[2])) 
                            elif '/' in revision_date:
                                rev_date_split = revision_date.split('/')
                                parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[2]), month=int(rev_date_split[1]), day=int(rev_date_split[0])) 
                                    
                        #get units
                        if 'units' in row[0].lower():
                            units = ','.join(row).replace('Units','').replace('units','').replace(',','').replace(':','').strip()
                            #replace micro sign if necessary
                            if u'\xb5' in units:
                                units = units.replace(u'\xb5','u')
                            #replace anomalous symbol which sometimes appears
                            if u'\xc2' in units:
                                units = units.replace(u'\xc2','') 
                            parsed_metadata['units'] = units
                            continue
                
                        #get column headers
                        if ('pollutant' in row[0].lower()) & ('H01' in row):
                            start_read = True
                            column_names = copy.deepcopy(row)
                            column_inds = {'date':column_names.index('Date'), 'H01':column_names.index('H01'), 'station_reference':column_names.index('NAPSID'), 'longitude':column_names.index('Longitude'), 'latitude':column_names.index('Latitude'), 'station_name':column_names.index('City')}
                            #check if have method codes
                            if 'Method' in column_names:
                                column_inds['method code'] = column_names.index('Method')
                                parsed_data['method code'] = []
                            elif 'MethodCode' in column_names:
                                column_inds['method code'] = column_names.index('MethodCode')
                                parsed_data['method code'] = []
                            continue

                        if start_read:

                            date = row[column_inds['date']] 
                            if '-' in date:
                                date_split = date.split('-')
                                date_year = date_split[0]
                                date_month = date_split[1]
                                date_day = date_split[2]
                            elif '/' in date:
                                date_split = date.split('/')
                                date_year = date_split[2]
                                date_month = date_split[0]
                                date_day = date_split[1]
                            else:
                                date_year = date[:4]
                                date_month = date[4:6]
                                date_day = date[6:]
                                
                            #iterate through hours
                            for hh in range(24):
            
                                parsed_data['start_dt'].append(datetime.datetime(year=int(date_year), month=int(date_month), day=int(date_day), hour=hh, minute=0))
                                parsed_data['end_dt'].append(parsed_data['start_dt'][-1] + relativedelta(hours=1))
                        
                                #measurements - fill values empty as empty str
                                meas = row[column_inds['H01']+hh]
                                if meas in ['-999','-9999']:
                                    parsed_data['measurement'].append('')
                                else:
                                    parsed_data['measurement'].append(meas)
                
                                #station reference                  
                                parsed_data['station_reference'].append(row[column_inds['station_reference']])
                                
                                #longitude/latitude
                                parsed_data['longitude'].append(row[column_inds['longitude']])
                                parsed_data['latitude'].append(row[column_inds['latitude']])

                                #station name
                                parsed_data['station_name'].append(row[column_inds['station_name']])

                                #method code
                                if 'method code' in column_inds:
                                    parsed_data['method code'].append(row[column_inds['method code']])

                    #add to all parsed lists
                    self.all_parsed_data.append(parsed_data)
                    self.all_parsed_metadata.append(parsed_metadata)   

        #xls/xlsx continuous file types
        elif self.file_type in ['xls','xlsx']:
            if self.file_type == 'xls':
                try:
                    sheets = pd.read_excel(self.filename, sheet_name=None, header=None, na_filter=False)
                except struct.error as err:
                    self.run_details['log_file_str'] += 'FILE IS CORRUPTED. CANNOT READ.\n'
                    return
                    
            elif self.file_type == 'xlsx':
                try:
                    sheets = pd.read_excel(self.filename, sheet_name=None, header=None, na_filter=False, engine='openpyxl')
                except struct.error as err:
                    self.run_details['log_file_str'] += 'FILE IS CORRUPTED. CANNOT READ.\n'
                    return                    

            #get data
            #only 1 sheet, then must be data on it
            if len(sheets.keys()) == 1:
                self.data_sheet_names = [list(sheets.keys())[0]]
                
            #iterate through all data sheets to process
            for data_sheet_name in self.data_sheet_names:
                if data_sheet_name not in sheets:
                    continue
                data_rows = sheets[data_sheet_name].values.tolist()

                parsed_metadata = {'data_sheet_name':data_sheet_name}
                parsed_data = {'start_dt':[],'end_dt':[],'measurement':[]}

                #get metadata
                metadata_sheet_name = ''
                if 'MetaData' in sheets.keys():
                    metadata_sheet_name = 'MetaData'
                elif 'Metadata' in sheets.keys():
                    metadata_sheet_name = 'Metadata'
                elif 'Metadata Information' in sheets.keys():
                    metadata_sheet_name = 'Metadata Information'
                if '/NAPSReferenceMethodPM2.5/' in self.filename:
                    metadata_sheet_name = copy.deepcopy(data_sheet_name)

                sampling_field = ''
                sampler_options = {}
                if metadata_sheet_name != '':
                    metadata_rows = sheets[metadata_sheet_name].values.tolist()
                    for metadata_row in metadata_rows:
                        if len(metadata_row) == 0:
                            continue
                        if '/NAPSReferenceMethodPM2.5/' in self.filename:
                            meta_key = metadata_row[3]
                            meta_value = metadata_row[4]
                        else:
                            meta_key = metadata_row[0]
                            meta_value = metadata_row[1]
                        #convert meta key/values NaNs to empty str
                        if pd.isnull(meta_key):
                            meta_key = '' 
                        else:
                            meta_key = str(meta_key).strip('*').strip().replace('/','_')
                        if pd.isnull(meta_value):
                            meta_value = ''
                        elif type(meta_value) != datetime.datetime:
                            meta_value = str(meta_value).strip()

                        #do variable correction for NAPSReferenceMethodPM2.5 files
                        if 'sampler #' in meta_key.lower():
                            meta_key = 'sampler' 
                            
                        #file revision date
                        if 'file generated' in meta_key.lower():
                            if type(meta_value) == datetime.datetime:
                                parsed_metadata['file_revision_date'] = meta_value
                            else:
                                if '-' in meta_value:
                                    rev_date_split = meta_value.split('-')
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[0]), month=int(rev_date_split[1]), day=int(rev_date_split[2]))
                                elif '/' in meta_value:
                                    rev_date_split = meta_value.split('/')
                                    if int(rev_date_split[2]) < 50:
                                        year_str = '20{}'.format(int(rev_date_split[2]))
                                    else:   
                                        year_str = '19{}'.format(int(rev_date_split[2]))
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(year_str), month=int(rev_date_split[1]), day=int(rev_date_split[0]))

                        #basic station meta
                        elif meta_key.lower() in ['longitude','latitude']:
                            if meta_key.lower() not in parsed_metadata:
                                parsed_metadata[meta_key.lower()] = meta_value
                        elif meta_key.lower() == 'elevation (m)':
                            if 'altitude' not in parsed_metadata:
                                parsed_metadata['altitude'] = meta_value
                        elif meta_key.lower() == 'probe height (m)':
                            if 'sampling_height' not in parsed_metadata:
                                parsed_metadata['sampling_height'] = meta_value
                        elif meta_key.lower() in ['site name','city']:
                            if 'station_name' not in parsed_metadata:
                                parsed_metadata['station_name'] = meta_value
                        elif meta_key.lower() == 'address':
                            if 'address' not in parsed_metadata:
                                parsed_metadata['address'] = meta_value
                        #station classifications
                        elif meta_key.lower() in ['spatial scale','land use','site type']:
                            meta_key_insert = meta_key.lower().replace(' ','_')
                            if meta_key_insert not in parsed_metadata:
                                parsed_metadata[meta_key_insert] = meta_value.split('-')[0].strip()

                        #sampling data - xlsx
                        elif 'Station Info' in sheets.keys():
                            if meta_key == '':
                                sampling_field = ''
                            elif meta_key.lower() in ['sampler','cartridge','medium','observation type','analytical instrument','sample preparation','method code']:
                                sampling_field = meta_key.lower()    
                                sampler_options[sampling_field] = {}
                            elif sampling_field != '':
                                sampler_options[sampling_field][meta_key] = meta_value      
                        #sampling data - xls
                        elif meta_key.lower() in ['sampler','cartridge','medium','observation type','analytical instrument','sample preparation','method code']:
                            if meta_key.lower() in parsed_metadata:
                                parsed_metadata[meta_key.lower()] = '{};{}'.format(parsed_metadata[meta_key.lower()],meta_value)
                            else:
                                parsed_metadata[meta_key.lower()] = meta_value
                
                #get station info
                if 'Station Info' in sheets.keys():
                    stninfo_rows = sheets['Station Info'].values.tolist()
                    sampler_prefix = ''
                    for stninfo_row in stninfo_rows:
                        if len(stninfo_row) == 0:
                            continue
                        meta_key = stninfo_row[0]
                        meta_value = stninfo_row[1]
                        #convert meta key/values NaNs to empty str
                        if pd.isnull(meta_key):
                            meta_key = '' 
                        else:
                            meta_key = meta_key.strip('*').strip()
                        if pd.isnull(meta_value):
                            meta_value = ''
                        elif type(meta_value) != datetime.datetime:
                            meta_value = str(meta_value).strip()
                    
                        #file revision date
                        if 'file generated' in meta_key.lower():
                            if type(meta_value) == datetime.datetime:
                                parsed_metadata['file_revision_date'] = meta_value
                            else:
                                if '-' in meta_value:
                                    rev_date_split = meta_value.split('-')
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(rev_date_split[0]), month=int(rev_date_split[1]), day=int(rev_date_split[2]))
                                elif '/' in meta_value:
                                    rev_date_split = meta_value.split('/')
                                    if int(rev_date_split[2]) < 50:
                                        year_str = '20{}'.format(int(rev_date_split[2]))
                                    else:   
                                        year_str = '19{}'.format(int(rev_date_split[2]))
                                    parsed_metadata['file_revision_date'] = datetime.datetime(year=int(year_str), month=int(rev_date_split[1]), day=int(rev_date_split[0]))

                        #basic station meta
                        elif meta_key.lower() in ['longitude','latitude']:
                            if meta_key.lower() not in parsed_metadata:
                                parsed_metadata[meta_key.lower()] = meta_value
                        elif meta_key.lower() == 'elevation (m)':
                            if 'altitude' not in parsed_metadata:
                                parsed_metadata['altitude'] = meta_value
                        elif meta_key.lower() == 'probe height (m)':
                            if 'sampling_height' not in parsed_metadata:
                                parsed_metadata['sampling_height'] = meta_value
                        elif meta_key.lower() in ['site name','city']:
                            if 'station_name' not in parsed_metadata:
                                parsed_metadata['station_name'] = meta_value
                        elif meta_key.lower() == 'address':
                            if 'address' not in parsed_metadata:
                                parsed_metadata['address'] = meta_value
                        #station classifications
                        elif meta_key.lower() in ['spatial scale','land use','site type']:
                            meta_key_insert = meta_key.lower().replace(' ','_')
                            if meta_key_insert not in parsed_metadata:
                                parsed_metadata[meta_key_insert] = meta_value.split('-')[0].strip()

                        #sampler number
                        elif 'Sampler #' in meta_key:
                            sampler_prefix = 'S-{}'.format(meta_key.split('#')[1][0])
                            if 'sampler' not in sampler_options: 
                                sampler_options['sampler'] = {}
                    
                        #sampler instrument name
                        elif (sampler_prefix != '') & (meta_key.lower() == 'description'): 
                            sampler_options['sampler'][sampler_prefix] = meta_value
                            sampler_prefix = ''
            
                #find column names line
                if '/NAPSReferenceMethodPM2.5/' in self.filename:
                    data_column_names_ind = 0
                else:
                    for data_row_ii, data_row in enumerate(data_rows): 
                        if ('NAPS ID' in data_row) or ('NAPS Site ID' in data_row):
                            data_column_names_ind = copy.deepcopy(data_row_ii) 
                            break

                data_column_names = [str(data_column_name) for data_column_name in data_rows[data_column_names_ind]]
                #set inds for desired fields to parse - multiple text variations for each parameter makes it hard work
                #for measurements can have multiple samples, so take first sample preferentially 
                data_column_inds = {}
                vars_to_test = {'station_reference':['NAPS ID','NAPS Site ID'],'date':['Sampling Date','Sample Date','Date','DATE','Sampling Date YYYY/MM/DD'], 'start_time':['Start Time','Start_time','START TIME','START_TIME'], 'end_time':['Stop Time','End Time','End_time','END_TIME'], 'duration':['DURATION'], 'sample_type':['Sample Type','Sampling Type'], 'media':['Media'], 'cartridge':['Cartridge','Cart'], 'pressure':['Pres.','pres.','PRESS','PRESSURE','pressure'], 'temperature':['Temp.','temp.','TEMP','TEMPERATURE','temperature'], 'observation_type':['C/F','Fraction','fraction'], 'measurement':[self.NAPS_code], 'data_flag':self.all_NAPS_codes, 'lod':self.all_NAPS_codes}
                #set measurement units initially as empty str
                units = ''
                for key,vals in vars_to_test.items():
                    for val in vals:
                        if key == 'measurement':
                            if key not in data_column_inds:
                                for data_column_ii, data_column_name in enumerate(data_column_names):
                                    if val == data_column_name:
                                        data_column_inds[key] = copy.deepcopy(data_column_ii) 
                                        break
                                    elif (val in data_column_name) & ('g/m3' in data_column_name):
                                        data_column_inds[key] = data_column_ii
                                        units = data_column_name.split(' ')[-1].strip().strip('()')
                                        break
                                    elif (val in data_column_name) & ('m-3' in data_column_name):
                                        data_column_inds[key] = data_column_ii
                                        units = data_column_name.split(' ')[-1].strip().strip('()')
                                        break
                                if key in data_column_inds:
                                    break 
                        elif key == 'data_flag':
                            if key not in data_column_inds:
                                if '{}-VFlag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-VFlag'.format(val))
                                    break  
                                elif '{}_VFlag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_VFlag'.format(val))
                                    break  
                                elif '{} VFlag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} VFlag'.format(val))
                                    break  
                                elif '{}-Vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-Vflag'.format(val))
                                    break  
                                elif '{}_Vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_Vflag'.format(val))
                                    break  
                                elif '{} Vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} Vflag'.format(val))
                                    break  
                                elif '{}-vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-vflag'.format(val))
                                    break  
                                elif '{}_vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_vflag'.format(val))
                                    break  
                                elif '{} vflag'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} vflag'.format(val))
                                    break  
                        elif key == 'lod':
                            if key not in data_column_inds:
                                if '{}-MDL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-MDL'.format(val))
                                    break  
                                elif '{}-DL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}-DL'.format(val))
                                    break  
                                elif '{}_MDL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_MDL'.format(val))
                                    break  
                                elif '{}_DL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{}_DL'.format(val))
                                    break  
                                elif '{} MDL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} MDL'.format(val))
                                    break  
                                elif '{} DL'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} DL'.format(val))
                                    break  
                                elif '{} D.L.'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} D.L.'.format(val))
                                    break  
                                elif '{} D.L'.format(val) in data_column_names:
                                    data_column_inds[key] = data_column_names.index('{} D.L'.format(val))
                                    break  
                        else:
                            if key not in data_column_inds:
                                if val in data_column_names:
                                    data_column_inds[key] = data_column_names.index(val)
                                    break     

                #if have not found measurement index, then move on to next data sheet 
                if 'measurement' not in data_column_inds:
                    self.all_parsed_data.append(parsed_data)
                    self.all_parsed_metadata.append(parsed_metadata) 
                    continue

                #if data_flag or lod not filled, make sure it is not stated in next 2 rightward columns
                if 'data_flag' not in data_column_inds:
                    column_start_ii = data_column_inds['measurement']+1
                    column_end_ii = data_column_inds['measurement']+3
                    for data_column_ii, data_column_name in enumerate(data_column_names[column_start_ii:column_end_ii]):
                        if data_column_name == 'VFlag':
                            data_column_inds['data_flag'] = column_start_ii + data_column_ii
                            break
                        elif data_column_name == 'Vflag':
                            data_column_inds['data_flag'] = column_start_ii + data_column_ii
                            break
                        elif data_column_name == 'vflag':
                            data_column_inds['data_flag'] = column_start_ii + data_column_ii
                            break

                if 'lod' not in data_column_inds:   
                    column_start_ii = data_column_inds['measurement']+1
                    column_end_ii = data_column_inds['measurement']+2
                    for data_column_ii, data_column_name in enumerate(data_column_names[column_start_ii:column_end_ii]):
                        if data_column_name == 'MDL':
                            data_column_inds['lod'] = column_start_ii + data_column_ii  
                            break
                        elif data_column_name == 'DL':
                            data_column_inds['lod'] = column_start_ii + data_column_ii 
                            break 
                        elif data_column_name == 'D.L.':
                            data_column_inds['lod'] = column_start_ii + data_column_ii  
                            break
                        elif data_column_name == 'D.L':
                            data_column_inds['lod'] = column_start_ii + data_column_ii     
                            break

                #do little extra work to get units if not already parsed
                #for XLS files --> on first line
                if self.file_type == 'xls':
                    if units == '':
                        #PMSpeciation files do not have any set units so assume they are ug/m3
                        if ('/PMSpeciation/' in self.filename) & (self.filename[-9:].lower() == '_spec.xls'):
                            units = 'ug/m3'
                        else:
                            #if have no set units, assume units for different file types
                            try:    
                                units = data_rows[0][0].split('(')[1].split(')')[0].strip()
                            except IndexError as error:
                                if ('/VOC/' in self.filename):
                                    units = 'ug/m3'
                                else:
                                    units = ''
                    if 'pressure' in data_column_inds:
                        pressure_units = 'kPa'
                    if 'temperature' in data_column_inds:
                        temperature_units = 'degC'
                    if 'lod' in data_column_inds:
                        lod_units = copy.deepcopy(units)
                #for xlsx files --> line above column names
                elif self.file_type == 'xlsx':
                    if units == '':
                        units = data_rows[data_column_names_ind-1][data_column_inds['measurement']]
                    if 'pressure' in data_column_inds:
                        pressure_units = data_rows[data_column_names_ind-1][data_column_inds['pressure']]
                    if 'temperature' in data_column_inds:
                        temperature_units = data_rows[data_column_names_ind-1][data_column_inds['temperature']]
                    if 'lod' in data_column_inds:
                        lod_units = data_rows[data_column_names_ind-1][data_column_inds['lod']]
                        #if lod units are empty str then assume they are same as measured units
                        if lod_units == '':
                            lod_units = copy.deepcopy(units) 
                #add units to parsed_metadata - convert micro sign if neccessary
                if u'\xb5' in units:
                    units = units.replace(u'\xb5','u')
                #replace anomalous symbol which sometimes appears
                if u'\xc2' in units:
                    units = units.replace(u'\xc2','') 
                parsed_metadata['units'] = units
                if 'pressure' in data_column_inds:
                    parsed_metadata['pressure_units'] = pressure_units
                if 'temperature' in data_column_inds:
                    #replace degree symbol if necessary
                    if u'\xb0' in temperature_units:
                        temperature_units = temperature_units.replace(u'\xb0','deg')
                    parsed_metadata['temperature_units'] = temperature_units
                if 'lod' in data_column_inds:
                    if u'\xb5' in lod_units:
                        lod_units = lod_units.replace(u'\xb5','u')
                    if u'\xc2' in lod_units:
                        lod_units = lod_units.replace(u'\xc2','') 
                    parsed_metadata['lod_units'] = lod_units

                #for xlsx files handle sampler and method information on data sheet
                #use sampler_options dict to parse sampler metadata
                if self.file_type == 'xlsx':
                    sampler_fields = ['sampler','cartridge','medium','observation type','analytical instrument','sample preparation','method code']
                    for data_row in data_rows[:data_column_names_ind]:
                        if len(data_row) == 0:
                            continue
                        meta_key = data_row[0]
                        meta_value = data_row[data_column_inds['measurement']]
                        if pd.isnull(meta_key):  
                            meta_key = ''
                        if pd.isnull(meta_value):
                            meta_value = ''  
                        if meta_key.lower() in sampler_fields:
                            if meta_key.lower() not in sampler_options:
                                parsed_metadata[meta_key.lower()] = meta_value
                            else:
                                if meta_value not in sampler_options[meta_key.lower()]:
                                    parsed_metadata[meta_key.lower()] = meta_value
                                else:
                                    parsed_metadata[meta_key.lower()] = sampler_options[meta_key.lower()][meta_value] 
            
                #iterate through rows
                for data_row in data_rows[data_column_names_ind+1:]:

                    if len(data_row) == 0:
                        continue

                    #get date - if missing then skip row
                    date = data_row[data_column_inds['date']]
                    str_date = str(date).strip()
                    if type(date) == int:
                        continue
                    if str_date in ['-999','-9999','','nan']:
                        continue
                    if type(date) == str:
                        archaic_date_format_eval = [True for month in list(self.month_dict.keys()) if month in str_date.upper()] 
                        if len(archaic_date_format_eval) > 0:
                            archaic_date_format = True
                        else:
                            archaic_date_format = False
                        if '-' in str_date:
                            date_split = str_date.split('-')
                            if len(date_split[0]) == 4:
                                year_n = 0
                                month_n = 1
                                day_n = 2
                            elif len(date_split[2]) == 4:
                                year_n = 2
                                month_n = 1
                                day_n = 0
                            elif archaic_date_format:
                                year_n = 2
                                month_n = 1
                                day_n = 0
                            else:
                                year_n = 0
                                month_n = 1
                                day_n = 2  
                            if archaic_date_format:
                                year_str = date_split[year_n]
                                month_str = self.month_dict[date_split[month_n].upper()]
                                day_str = date_split[day_n]
                            else:
                                year_str = date_split[year_n]
                                month_str = date_split[month_n]
                                day_str = date_split[day_n]

                        elif '/' in str_date:
                            date_split = str_date.split('/')
                            if len(date_split[0]) == 4:
                                year_n = 0
                                month_n = 1
                                day_n = 2
                            elif len(date_split[2]) == 4:
                                year_n = 2
                                month_n = 0
                                day_n = 1
                            elif archaic_date_format:
                                year_n = 2
                                month_n = 1
                                day_n = 0
                            else:
                                year_n = 2
                                month_n = 0
                                day_n = 1  
                            if archaic_date_format:
                                year_str = date_split[year_n]
                                month_str = self.month_dict[date_split[month_n].upper()]
                                day_str = date_split[day_n]
                            else:
                                year_str = date_split[year_n]
                                month_str = date_split[month_n]
                                day_str = date_split[day_n]
                        else:
                            continue

                        #test that year, month, day are numbers
                        try:
                            int_year_str = int(year_str)
                            int_month_str = int(month_str)
                            int_day_str = int(day_str)
                        except ValueError as err:
                            continue
 
                        #need to make year 4 characters?
                        if len(year_str) == 2:
                            if int(year_str) < 50:
                                year_str = '20{}'.format(year_str)
                            else:   
                                year_str = '19{}'.format(year_str)  
       
                        #check parsed year is same as data year, otherwise continue
                        if self.data_year != year_str:
                            continue

                        #check date is valid, otherwise continue
                        try:
                            date = datetime.datetime(year=int(year_str),month=int(month_str),day=int(day_str))
                        except ValueError as err:
                            continue

                    #check parsed year is same as data year, otherwise continue
                    if int(self.data_year) != date.year:
                        continue

                    have_time = False

                    #time - have start time and duration
                    #if either of start time or duration are missing then assume date is for whole day (unless stated in filename)
                    if ('start_time' in data_column_inds) & ('duration' in data_column_inds): 
                        start_time = str(data_row[data_column_inds['start_time']]).strip()
                        duration = str(data_row[data_column_inds['duration']]).strip()
    
                        have_time = True                    
                        for test_var in [start_time,duration]:
                            if test_var in ['-999','-9999','','nan','???','?']:
                                have_time = False
                                break

                        if have_time:
                            if ':' in start_time:
                                st_split = start_time.split(':')
                            elif ';' in start_time:
                                st_split = start_time.split(';')
                            elif '.' in start_time:
                                start_time = str(round(float(start_time),2))
                                st_split = start_time.split('.')
                            start_hr = int(st_split[0])
                            start_min = int(st_split[1])
                            if '.' in duration:
                                duration = str(round(float(duration),2))
                                dur_split = duration.split('.')
                                duration_hours = int(dur_split[0])
                                duration_mins = int(dur_split[1])
                            elif ':' in duration:
                                dur_split = duration.split(':')
                                duration_hours = int(dur_split[0])
                                duration_mins = int(dur_split[1])                                
                            elif 'hr' in duration:
                                dur_split = duration.split(' ')
                                duration_hours = int(dur_split[0].replace('hr',''))
                                duration_mins = int(dur_split[1].replace('min',''))
                            else:    
                                duration_hours = int(duration)
                                duration_mins = 0
                            parsed_data['start_dt'].append(date.replace(hour=start_hr,minute=start_min))
                            parsed_data['end_dt'].append(parsed_data['start_dt'][-1] + relativedelta(hours=duration_hours,minutes=duration_mins))  
                                                      
                    #time - have start and end time
                    #if either of start time or end_time are missing then assume date is for whole day (unless stated in filename)
                    elif ('start_time' in data_column_inds) & ('end_time' in data_column_inds):
                        start_time = str(data_row[data_column_inds['start_time']]).strip()
                        end_time = str(data_row[data_column_inds['end_time']]).strip()
                        
                        #if start or end time is empty then continue to next line
                        have_time = True
                        for test_var in [start_time,end_time]:
                            if test_var in ['-999','-9999','','nan']:
                                have_time = False
                                break

                        if have_time:
                            if ':' in start_time:
                                st_split = start_time.split(':')
                            elif ';' in start_time:
                                st_split = start_time.split(';')
                            elif '.' in start_time:
                                start_time = str(round(float(start_time),2))
                                st_split = start_time.split('.')
                            if ':' in end_time:
                                et_split = end_time.split(':')
                            elif ';' in end_time:
                                et_split = end_time.split(';')
                            elif '.' in end_time:
                                end_time = str(round(float(end_time),2))
                                et_split = end_time.split('.')
                            start_hr = int(st_split[0])
                            start_min = int(st_split[1])
                            end_hr = int(et_split[0])
                            end_min = int(et_split[1])
                            parsed_data['start_dt'].append(date.replace(hour=start_hr,minute=start_min)) 
                            if end_hr >= 24:
                                days_to_add = end_hr//24
                                end_hr = end_hr - (days_to_add*24) 
                                parsed_data['end_dt'].append((parsed_data['start_dt'][-1] + relativedelta(days=days_to_add)).replace(hour=end_hr,minute=end_min))
                            else:
                                parsed_data['end_dt'].append(date.replace(hour=end_hr,minute=end_min)) 
                                
                    #no time - so assume daily sample (unless stated in data filename)
                    if not have_time:
                        if '_4hr_' in self.filename:
                            parsed_data['start_dt'].append(date.replace(hour=12))
                            parsed_data['end_dt'].append(date.replace(hour=16))
                        else:
                            parsed_data['start_dt'].append(date)
                            parsed_data['end_dt'].append(parsed_data['start_dt'][-1] + relativedelta(days=1))  

                    #add rest of variables
                    for var in data_column_inds.keys():
                        if var not in ['date','start_time','end_time','duration']:
                            if var not in parsed_data:
                                parsed_data[var] = []
                            val = str(data_row[data_column_inds[var]]).strip()
                            if val in ['-999','-9999','','nan']:
                                parsed_data[var].append('')
                            else:
                                parsed_data[var].append(val)

                #add to all parsed lists
                self.all_parsed_data.append(parsed_data)
                self.all_parsed_metadata.append(parsed_metadata)    

        #convert lists to numpy arrays
        for parsed_data_ii, parsed_data in enumerate(self.all_parsed_data):
            for column_header in list(parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.all_parsed_data[parsed_data_ii][column_header] = np.array(parsed_data[column_header])
                else:
                    self.all_parsed_data[parsed_data_ii][column_header] = np.array(parsed_data[column_header], dtype=np.str_) 
     

class ParseCAPMoNFile:

    '''class that handles processing of a CAPMoN file'''

    def __init__(self,filename, parameter, parameter_details):

        self.filename = filename
        self.parameter = parameter
        self.parameter_details = parameter_details

        self.parsed_metadata = {}
        self.parsed_detection_limits = {'siteid':[], 'year':[], 'instrument_detection_limit':[]}
        if self.parameter == 'O3':
            self.parsed_data = {'siteid':[], 'start_dt':[], 'end_dt':[], 'measurements':[], 'flags':[], 'lods':[]}
        elif 'PREC' in self.parameter:
            self.parsed_data = {'siteid':[], 'start_dt':[], 'end_dt':[], 'measurements':[], 'flags':[], 'lods':[]}
        else:
            self.parsed_data = {'siteid':[], 'start_dt':[], 'end_dt':[], 'measurements':[], 'flags':[]}
            
        if self.parameter == 'PM10_Co':
            self.lod_ref = 'COB_NGM3'
            self.flag_ref = 'COB_NGM3_Flag'
            self.meas_ref = 'COB_NGM3'
        elif self.parameter == 'PM10_Zn':
            self.lod_ref = 'ZN_NGM3'
            self.flag_ref = 'Zn_NGM3_Flag'
            self.meas_ref = 'Zn_NGM3'
        elif self.parameter == 'PREC_ACCUM':
            self.lod_ref = 'StandardGaugeDeptH_mm_DetL'
            self.flag_ref = 'StandardGaugeDeptH_mm_Flag'
            self.meas_ref = 'StandardGaugeDeptH_mm'
        elif '_PREC' in self.parameter:
            self.lod_ref = self.parameter_details['chemical_formula']+'_mgL_DetL'
            self.flag_ref = self.parameter_details['chemical_formula']+'_mgL_Flag'
            self.meas_ref = self.parameter_details['chemical_formula']+'_mgL'   
        else:
            self.lod_ref = '{}_NGM3'.format(self.parameter_details['chemical_formula'].upper())
            self.flag_ref = '{}_NGM3_Flag'.format(self.parameter_details['chemical_formula'].upper())
            self.meas_ref = '{}_NGM3'.format(self.parameter_details['chemical_formula'].upper())

    def read_data(self):

        active_meta = False
        active_table = False
        table_name = ''
         
        with open(self.filename, newline='', encoding='latin1') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"', skipinitialspace=True)
            for row_count, row in enumerate(csvreader):
            
                #empty row
                if len(row) == 0:
                    continue

                #remove french translation from row columns
                row = [r.split('//')[0].strip() for r in row]

                #file revision date?
                if row[0] == '*DATE THIS FILE GENERATED':
                    self.parsed_metadata['file_revision_date'] = [row[1]]

                # get analytical method per component
                if row[0] == '*TABLE COLUMN LABORATORY ANALYTICAL METHOD':
                    analytical_methods = row[1:]

                #beginning of data table?
                if row[0] == '*TABLE NAME':
                    #site information
                    if row[1] == 'Site information':
                        table_name = 'site_information'
                    #instrument detection limits
                    elif row[1] == 'Instrument Detection Limits':
                        table_name = 'instrument_detection_limits'
                    #data
                    elif row[1] in ['CAPMoN_Ozone','GLBM METALS', 'CAPMoNPrecipitationChemistry', 'CAPMoN_Precip_Chemistry']:
                        table_name = 'data'
                    else:
                        table_name = ''

                    if table_name != '':
                        active_meta = True
                    continue

                #start of data table?
                if (row[0] == '*TABLE DATA BEGINS') & (table_name != ''):
                    active_meta = False
                    active_table = True
                    continue

                #end of data table?
                if row[0] == '*TABLE DATA ENDS':
                    active_table = False

                #active table meta?
                if active_meta:
                    #fill codes
                    if row[0] == '*TABLE COLUMN MISSING CODE':
                        missing_codes = row[1:]
                        #transform '-999.' to '-999.0' to correctly catch missing codes
                        missing_codes = ['-999.0' if missing_code == '-999.' else missing_code for missing_code in missing_codes]

                    #column names
                    if row[0] == '*TABLE COLUMN NAME--SHORT FORM': 
                        column_names_raw = row[1:]
                        column_names = []
                        for column_name_raw in column_names_raw: # do corrections for H+
                            column_names.append(column_name_raw.replace('__', '_').replace('h_', 'H_'))

                        if table_name == 'site_information':
                            for column_name in column_names:
                                if column_name != '':
                                    if column_name == 'SiteID':
                                        self.parsed_metadata['siteid'] = []
                                    else:
                                        self.parsed_metadata[column_name.lower()] = []    

                #parse data line
                if active_table:
                    row = row[1:]

                    # check if header and data have same length
                    if len(row) != len(missing_codes):
                        n_diff_columns = abs(len(row) - len(missing_codes))
                        missing_codes = missing_codes[:-n_diff_columns] # cut last column of header

                    #replace fill values with empty strings
                    for column_ii, missing_code in enumerate(missing_codes):
                        if row[column_ii] == missing_code:
                            row[column_ii] = ''

                    # check if header and data have same length
                    if len(row) != len(column_names):
                        n_diff_columns = abs(len(row) - len(column_names))
                        column_names = column_names[:-n_diff_columns] # cut last column of header
                    
                    #metadata
                    if table_name == 'site_information':
                        for column_name in column_names:
                            if column_name != '':
                                if column_name == 'SiteName':
                                    self.parsed_metadata[column_name.lower()].append(row[column_names.index(column_name)].split(';')[0])
                                elif column_name == 'SiteID':
                                    self.parsed_metadata['siteid'].append(row[column_names.index(column_name)])
                                else:
                                    self.parsed_metadata[column_name.lower()].append(row[column_names.index(column_name)])
                                    
                    elif table_name == 'instrument_detection_limits':
                        #siteid
                        self.parsed_detection_limits['siteid'].append(row[column_names.index('SiteID')])
                        #year
                        self.parsed_detection_limits['year'].append(row[column_names.index('Year')])
                        #detection limit
                        if self.parameter != 'H+_PREC':
                            self.parsed_detection_limits['instrument_detection_limit'].append(row[column_names.index(self.lod_ref)])
                        else: # no detection limit given for H+
                            self.parsed_detection_limits['instrument_detection_limit'].append('') #[0.0]*len(row[column_names.index('SiteID')])) # ????
                        
                    #measurement data
                    elif table_name == 'data':

                        # Analytical method: only parse the one for parameter
                        self.parsed_metadata['analytical_method'] = analytical_methods[column_names.index(self.meas_ref)]

                        #SiteID 
                        self.parsed_data['siteid'].append(row[column_names.index('SiteID')])

                        #parse datetime
                        start_date = row[column_names.index('DateStartUTC')].split('-')
                        start_time = row[column_names.index('TimeStartUTC')].split(':')
                        end_date = row[column_names.index('DateEndUTC')].split('-')
                        end_time = row[column_names.index('TimeEndUTC')].split(':')

                        #O3
                        if self.parameter == 'O3':
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[0]),month=int(start_date[1]),day=int(start_date[2]),hour=int(start_time[0]),minute=int(start_time[1]),second=int(start_time[2])))
                            self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[0]),month=int(end_date[1]),day=int(end_date[2]),hour=int(end_time[0]),minute=int(end_time[1]),second=int(end_time[2])))
                            self.parsed_data['measurements'].append(row[column_names.index('O3_ppbv')])
                            self.parsed_data['flags'].append(row[column_names.index('O3_ppbv_Flag')])
                            self.parsed_data['lods'].append(row[column_names.index('O3_ppbv_DetL')])

                        #precip
                        elif 'PREC' in self.parameter:
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[0]),month=int(start_date[1]),day=int(start_date[2]),hour=int(start_time[0]),minute=int(start_time[1]),second=int(start_time[2])))
                            self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[0]),month=int(end_date[1]),day=int(end_date[2]),hour=int(end_time[0]),minute=int(end_time[1]),second=int(end_time[2])))
                            self.parsed_data['measurements'].append(row[column_names.index(self.meas_ref)])
                            self.parsed_data['flags'].append(row[column_names.index(self.flag_ref)])
                            self.parsed_data['lods'].append('') # detection limits too variable to pass data quality checks in S3 -> do not parse

                            # if self.parameter != 'H+_PREC':
                            #     #self.parsed_data['lods'].append(row[column_names.index(self.lod_ref)])
                            #     self.parsed_data['lods'].append('')
                            # else: # no detection limit given
                            #     self.parsed_data['lods'].append('') #[0.0]*len(row[column_names.index('SiteID')])) # ??? 

                        #particulate metals
                        else:
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[0]),month=int(start_date[1]),day=int(start_date[2]),hour=int(start_time[0]),minute=int(start_time[1])))
                            self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[0]),month=int(end_date[1]),day=int(end_date[2]),hour=int(end_time[0]),minute=int(end_time[1])))
                            self.parsed_data['measurements'].append(row[column_names.index(self.meas_ref)])
                            self.parsed_data['flags'].append(row[column_names.index(self.flag_ref)])
                         

        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 
        for column_header in list(self.parsed_metadata.keys()):
            self.parsed_metadata[column_header] = np.array(self.parsed_metadata[column_header], dtype=np.str_) 
        for column_header in list(self.parsed_detection_limits.keys()):
            self.parsed_detection_limits[column_header] = np.array(self.parsed_detection_limits[column_header], dtype=np.str_)

        #return self.parsed_data
        

class ParseCHILESINCAFile:

    '''class that handles processing of a CHILE SINCA file'''

    def __init__(self,filename):

        self.filename = filename
        temporal_resolution = self.filename.split('_')[-1].split('.')[0]
        if temporal_resolution == 'hourly':
            self.timedelta = relativedelta(hours=1)
        elif temporal_resolution == 'daily':
            self.timedelta = relativedelta(days=1)
        elif temporal_resolution == 'monthly':
            self.timedelta = relativedelta(months=1)

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=';', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers if column_header not in ['FECHA (YYMMDD)','HORA (HHMM)']}
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        if column_header == 'FECHA (YYMMDD)':
                            line_year = element[:2]
                            if int(line_year) >= 50:
                                line_year = '19'+line_year
                            else:
                                line_year = '20'+line_year
                            line_month = element[2:4]
                            line_day = element[4:6] 
                            line_hour = row[column_headers.index('HORA (HHMM)')][:2]
                            line_minute = row[column_headers.index('HORA (HHMM)')][2:4]
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(line_year),month=int(line_month),day=int(line_day),hour=int(line_hour),minute=int(line_minute))) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + self.timedelta)
                        elif column_header != 'HORA (HHMM)':
                            element = element.replace(',','.')
                            self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 
        
        return self.parsed_data


class ParseCNEMCFile:

    '''class that handles processing of a CNEMC file'''

    def __init__(self,filename,run_details):

        self.filename = filename
        self.run_details = run_details

    def read_data(self):

        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')

            for row_count, row in enumerate(csvreader):

                #column headers
                # need to get unique column headers as can have a few station column headers which are repeated 
                # in these cases just keep first instance
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    unique_column_headers = np.unique(column_headers)
                    header_indices = {column_header:column_headers.index(column_header) for column_header in unique_column_headers} 
                    self.parsed_data = {column_header:[] for column_header in unique_column_headers if column_header not in ['date','hour','type']}
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    #make sure number of columns on line is same as number of column headers, if not then skip line
                    if len(row) == len(column_headers): 
                        param = row[header_indices['type']]
                        #only parse data for current parameter
                        if param == self.run_details['parameter']:
                            #parse date
                            date = row[header_indices['date']]
                            year = int(date[:4])
                            month = int(date[4:6])
                            day = int(date[6:])
                            hour = int(row[header_indices['hour']])
                            self.parsed_data['start_dt'].append(datetime.datetime(year=year,month=month,day=day,hour=hour,minute=0)) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                            #parse station measurements
                            for column_header in unique_column_headers:
                                if column_header not in ['date','hour','type']:
                                    self.parsed_data[column_header].append(row[header_indices[column_header]])
                            
            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 


class ParseEANETFile:

    '''class that handles processing of a EANET file'''

    def __init__(self,filename, run_details):

        self.filename = filename
        self.run_details = run_details

    def read_data(self):

        with open(self.filename, newline='', encoding="cp850") as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            # count lines of header
            header_line_count_active = False
            indices_data_table = []
            start_the_table=False
            start_data=False

            for row_count, row in enumerate(csvreader):

                if 'PREC' in self.run_details['parameter']: # prec files

                    if any('Form' in s for s in row): # identify first line
                        self.parsed_data = {'start_dt':[], 'end_dt':[], self.run_details['parameter']:[], 'flag1':[], 'flag2':[], 'flag3':[], 'method_code':[]}
                        indices_data_table = [i for i, e in enumerate(row) if ('Form' in e)]
                        start_the_table=True
                        
                    # cut out data table, skip empty columns in the beginning if existing
                    if start_the_table:
                        row = row[indices_data_table[0]:]

                        # strip header to be in one line, in case it is not
                        if row[0] == 'Sample No.':
                            header_line_count_active = True
                            header = row

                        if row[1] == 'Start':
                            header_line_count_active = False

                        # append rows that make up header
                        if header_line_count_active:
                            header.append(row)
                            
                        if row[1] == 'Start': # header is ready
                            start_data=True

                            # cut tables
                            indices_tables = [i for i, e in enumerate(header) if ('Sample No.' in e)]

                            # cut header
                            if '-' in self.run_details['parameter']:
                                header = header[indices_tables[0]:indices_tables[1]]
                            elif ('+' in self.run_details['parameter']) and (self.run_details['parameter'] != 'H+_PREC'):
                                header = header[indices_tables[1]:indices_tables[2]]
                            elif self.run_details['parameter'] in ['PREC_ACCUM', 'H+_PREC']:
                                header = header[indices_tables[2]:indices_tables[3]]
                            else:
                                continue

                            # find header and correct names (row is new defined!)
                            to_correct = {'Ca2+':'Ca++_PREC','Mg2+':'Mg++_PREC','K+':'K+_PREC','Na+':'Na+_PREC','SO42-':'SO4--_PREC', 'Cl-':'Cl-_PREC', 'NO3-':'NO3-_PREC', 'NH4+':'NH4+_PREC','F-':'F-_PREC','PO43-':'PO4---_PREC', 'pH':'H+_PREC', 'Amountofprecipitation': 'PREC_ACCUM'}
                            #if row[0] == 'Sample No.':
                            column_headers = [element.replace(' ', '') for element in header]
                            for to_c in to_correct.keys():
                                if to_c in column_headers:
                                    column_headers[column_headers.index(to_c)] = to_correct[to_c]
                        
                        if start_data:
                            # cut tables
                            if '-' in self.run_details['parameter']:
                                row = row[indices_tables[0]:indices_tables[1]] # table of Anions
                            elif ('+' in self.run_details['parameter']) and (self.run_details['parameter'] != 'H+_PREC'):
                                row = row[indices_tables[1]:indices_tables[2]] # table of Cations
                            elif self.run_details['parameter'] in ['PREC_ACCUM', 'H+_PREC']:
                                row = row[indices_tables[2]:indices_tables[3]] # table of precip
                            else:
                                continue

                            # redefined row
                            if len(row) > 0: # no data for that table
                                if row[0].strip().isdigit(): #sample numbers

                                    if not any(s.strip() for s in row[1:]): # skip empty rows
                                        continue

                                    # dates
                                    if (row[1] != '') or (row[3] != ''):
                                        #start
                                        split_date = row[1].replace('//', '/').split('/')
                                        if len(split_date) > 0: # skip empty rows
                                            if len(split_date[0]) == 4: # format YYYYMMDD
                                                start_year = split_date[0]
                                                start_month = split_date[1]
                                                if len(split_date) == 3:
                                                    start_day = split_date[2]
                                                else:
                                                    start_day = '0'
                                            else: # format MMDDYYYY
                                                start_year = split_date[2]
                                                start_month = split_date[0]
                                                start_day = split_date[1]
                                        else:
                                            start_year = '0'
                                            start_month = '0'
                                            start_day = '0'

                                        #end
                                        split_date = row[3].replace('//', '/').split('/')
                                        if len(split_date) > 0: # skip empty rows
                                            if len(split_date[0]) == 4:
                                                end_year = split_date[0]
                                                end_month = split_date[1]
                                                if len(split_date) == 3:
                                                    end_day = split_date[2]
                                                else:
                                                    end_day = '0'
                                            else:
                                                end_year = split_date[2]
                                                end_month = split_date[0]
                                                end_day = split_date[1]
                                        else:
                                            start_year = '0'
                                            start_month = '0'
                                            start_day = '0'

                                        # hours
                                        if row[2] != '':
                                            start_hour = row[2].replace('.', ':').split(':')[0]
                                            if len(row[2].split(':')) == 2:
                                                start_minute = row[2].replace('.', ':').split(':')[1]
                                            else:
                                                start_minute = '0'
                                        else:
                                            start_minute = '0'
                                            start_hour = '0'

                                        if row[4] != '':
                                            end_hour = row[4].replace('.', ':').split(':')[0]
                                            if len(row[4].split(':')) == 2:
                                                end_minute = row[4].replace('.', ':').split(':')[1]
                                            else:
                                                end_minute = '0'
                                        else:
                                            end_hour = '0'
                                            end_minute = '0'
                                    else:
                                        continue

                                    # catch erronous dates
                                    if start_year == '1900':
                                        start_year = '2021'
                                        start_month = '4'
                                        start_day = '15'

                                    self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_year),month=int(start_month),day=int(start_day),hour=int(start_hour),minute=int(start_minute)))
                                    self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_year),month=int(end_month),day=int(end_day),hour=int(end_hour),minute=int(end_minute)))

                                    for column_header in column_headers:
                                        if self.run_details['parameter'] == column_header:
                                            # measurements
                                            if 'H+_PREC' == self.run_details['parameter']:
                                                if row[column_headers.index(column_header)] == '':
                                                    meas = ''
                                                else:
                                                    meas = 10**-float(row[column_headers.index(column_header)]) * 10**6 # convert pH to H+ concentration in umol/l as it is linear
                                            else:
                                                meas = row[column_headers.index(column_header)]

                                            self.parsed_data[column_header].append(meas)

                                            # flags
                                            if 'PREC_ACCUM' == self.run_details['parameter']:
                                                self.parsed_data['method_code'].append(row[column_headers.index(column_header)+1])
                                                for n_flags in [1,2,3]: # parse up to 3 flags if existent
                                                    try:
                                                        # can flag be an integer? if yes, parse it
                                                        raw_flag = int(row[column_headers.index(column_header)+n_flags+1])
                                                        self.parsed_data['flag'+str(n_flags)].append(str(raw_flag))
                                                    except:
                                                        raw_flag = ''
                                                        self.parsed_data['flag'+str(n_flags)].append(str(raw_flag))
                                                        
                                            else:
                                                for n_flags in [1,2,3]: # parse up to 3 flags if existent
                                                    try:
                                                        # can flag be an integer? if yes, parse it
                                                        raw_flag = int(row[column_headers.index(column_header)+n_flags])
                                                        self.parsed_data['flag'+str(n_flags)].append(str(raw_flag))
                                                    except:
                                                        raw_flag = ''
                                                        self.parsed_data['flag'+str(n_flags)].append(str(raw_flag))


                else: # not precip files

                    #parameter units
                    if row_count == 0:
                        row_units = [unit.strip().encode("ascii", "ignore").decode() for unit in row]
                    #column headers
                    elif row_count == 1:
                        column_headers = [column_header.strip().encode("ascii", "ignore").decode() for column_header in row]

                        #correct certain column header names 
                        to_correct = {'Ca2+':'Ca++','Mg2+':'Mg++','SO42-':'SO4--'}
                        for to_c in to_correct.keys():
                            if to_c in column_headers:
                                column_headers[column_headers.index(to_c)] = to_correct[to_c]

                        #check if date/time is repeated if so, modify column names
                        if column_headers.count('Date') >= 2:
                            column_headers[column_headers.index('Date')] = 'Start_Date'
                            column_headers[column_headers.index('Time')] = 'Start_Time'
                            column_headers[column_headers.index('Date')] = 'End_Date'
                            column_headers[column_headers.index('Time')] = 'End_Time'
                        self.parsed_data = {column_header:[] for column_header in column_headers if column_header not in ['Start_Date','Start_Time','End_Date','End_Time','Year','Month','Day','Hour']}
                        self.parsed_data['start_dt'] = []
                        self.parsed_data['end_dt'] = []
                        self.units = {column_header:row_unit for row_unit,column_header in zip(row_units,column_headers) if (column_header not in ['Country','Site','Start_Date','Start_Time','End_Date','End_Time','Year','Month','Day','Hour','Memo','Note']) & (row_unit != 'Unit')}
                    #data lines
                    else:
                        #remove non-ascii characters from row
                        row = [element.strip().encode("ascii", "ignore").decode() for element in row]

                        #do not append line if row all blank, or if have missing key time variables
                        all_blank = all('' == r for r in row)
                        if all_blank:
                            continue

                        #check and process time information       
                        #non-automatic time
                        if 'Start_Date' in column_headers:
                            if (row[column_headers.index('Start_Date')] == '') or (row[column_headers.index('End_Date')] == ''):
                                continue
                            start_date_split = row[column_headers.index('Start_Date')].split('/')
                            end_date_split = row[column_headers.index('End_Date')].split('/')
                            start_time_split = row[column_headers.index('Start_Time')].split(':')
                            end_time_split = row[column_headers.index('End_Time')].split(':')

                            #no start date column, skip line
                            if (len(start_date_split) == 1) & (start_date_split[0] == ''):
                                continue
                            #no month or day, assume XXXX:01:01
                            elif len(start_date_split) == 1:
                                line_year_start = start_date_split[0]
                                line_month_start = '01'
                                line_day_start = '01'
                            #no day, assume XXXX:XX:00
                            elif len(start_date_split) == 2:
                                line_year_start = start_date_split[0]
                                line_month_start = start_date_split[1]
                                line_day_start = '01'
                            #all date information present
                            else:
                                line_year_start = start_date_split[0]
                                line_month_start = start_date_split[1]
                                line_day_start = start_date_split[2] 

                            #no end date column, skip line
                            if (len(end_date_split) == 1) & (end_date_split[0] == ''):
                                continue
                            #no month or day, assume XXXX:01:01
                            elif len(end_date_split) == 1:
                                line_year_end = end_date_split[0]
                                line_month_end = '01'
                                line_day_end = '01'
                            #no day, assume XXXX:XX:00
                            elif len(end_date_split) == 2:
                                line_year_end = end_date_split[0]
                                line_month_end = end_date_split[1]
                                line_day_end = '01'
                            #all date information present
                            else:
                                line_year_end = end_date_split[0]
                                line_month_end = end_date_split[1]
                                line_day_end = end_date_split[2] 

                            #no start time column, assume 00:00:00
                            if (len(start_time_split) == 1) & (start_time_split[0] == ''):
                                line_hour_start = '00'
                                line_minute_start = '00'
                                line_second_start = '00'
                            #no minutes or seconds, assume XX:00:00
                            elif len(start_time_split) == 1:
                                line_hour_start = start_time_split[0]
                                line_minute_start = '00'
                                line_second_start = '00'
                            #no seconds, assume XX:XX:00
                            elif len(start_time_split) == 2:
                                line_hour_start = start_time_split[0]
                                line_minute_start = start_time_split[1]
                                line_second_start = '00'
                            #all time information present
                            else:
                                line_hour_start = start_time_split[0]
                                line_minute_start = start_time_split[1]
                                line_second_start = start_time_split[2] 

                            #no end time column, assume 00:00:00
                            if (len(end_time_split) == 1) & (end_time_split[0] == ''):
                                line_hour_end = '00'
                                line_minute_end = '00'
                                line_second_end = '00'
                            #no minutes or seconds, assume XX:00:00
                            elif len(end_time_split) == 1:
                                line_hour_end = end_time_split[0]
                                line_minute_end = '00'
                                line_second_end = '00'
                            #no seconds, assume XX:XX:00
                            elif len(end_time_split) == 2:
                                line_hour_end = end_time_split[0]
                                line_minute_end = end_time_split[1]
                                line_second_end = '00'
                            #all time information present
                            else:
                                line_hour_end = end_time_split[0]
                                line_minute_end = end_time_split[1]
                                line_second_end = end_time_split[2]    

                            #check date/time is sensible before generating datetimes, if not do not process line
                            if (len(line_year_start) != 4) or (len(line_month_start) > 2) or (len(line_day_start) > 2) or (len(line_hour_start) > 2) or (len(line_minute_start) > 2) or (len(line_second_start) > 2):
                                continue
                            if (len(line_year_end) != 4) or (len(line_month_end) > 2) or (len(line_day_end) > 2) or (len(line_hour_end) > 2) or (len(line_minute_end) > 2) or (len(line_second_end) > 2):
                                continue

                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(line_year_start),month=int(line_month_start),day=int(line_day_start),hour=int(line_hour_start),minute=int(line_minute_start),second=int(line_second_start)))
                            self.parsed_data['end_dt'].append(datetime.datetime(year=int(line_year_end),month=int(line_month_end),day=int(line_day_end),hour=int(line_hour_end),minute=int(line_minute_end),second=int(line_second_end)))

                        #automatic time
                        else:
                            #if have no year then do not append line
                            if row[column_headers.index('Year')] == '':
                                continue
                            #if have no month then assume measurement is annual average
                            elif 'Month' not in column_headers:
                                line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=1,day=1,hour=0,minute=0)
                                line_end_dt = line_start_dt + relativedelta(years=1)
                            elif row[column_headers.index('Month')] == '':
                                line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=1,day=1,hour=0,minute=0)
                                line_end_dt = line_start_dt + relativedelta(years=1)
                            #if have no day then assume measurement is monthly average
                            elif 'Day' not in column_headers:
                                line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=1,hour=0,minute=0)
                                line_end_dt = line_start_dt + relativedelta(months=1)
                            elif row[column_headers.index('Day')] == '':
                                line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=1,hour=0,minute=0)
                                line_end_dt = line_start_dt + relativedelta(months=1)
                            #if have no hour then assume measurement is daily average
                            elif 'Hour' not in column_headers:
                                line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=int(row[column_headers.index('Day')]),hour=0,minute=0)
                                line_end_dt = line_start_dt + relativedelta(days=1)
                            elif row[column_headers.index('Hour')] == '':
                                line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=int(row[column_headers.index('Day')]),hour=0,minute=0)
                                line_end_dt = line_start_dt + relativedelta(days=1)
                            #otherwise have hourly data
                            else:
                                line_start_dt = datetime.datetime(year=int(row[column_headers.index('Year')]),month=int(row[column_headers.index('Month')]),day=int(row[column_headers.index('Day')]),hour=int(row[column_headers.index('Hour')])-1,minute=0)
                                line_end_dt = line_start_dt + relativedelta(hours=1)                       

                            #do check to ensure datetimes are within a sensible range (1900-2100)
                            if (line_start_dt < datetime.datetime(1900,1,1)) or (line_end_dt < datetime.datetime(1900,1,1)) or (line_start_dt > datetime.datetime(2100,1,1)) or (line_end_dt > datetime.datetime(2100,1,1)):
                                continue

                            self.parsed_data['start_dt'].append(line_start_dt)
                            self.parsed_data['end_dt'].append(line_end_dt)

                        #append rest of desired line values
                        for column_header, element in zip(column_headers, row):
                            if column_header in list(self.parsed_data.keys()):
                                self.parsed_data[column_header].append(element) 


        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)

        return self.parsed_data


class ParseEEAAIRBASEFile:

    '''class that handles processing of a EEA AIRBASE file'''

    def __init__(self,filename):

        self.filename = filename
        filename_split = self.filename.split('/')[-1]
        self.component_code = filename_split[7:12]
        self.measurement_european_group_code = filename_split[12:17]
        self.temporal_resolution = filename_split[17:].split('.')[0]
        self.data_file_start_date = datetime.datetime(int(filename_split.split('.')[-2].split('-')[2]), int(filename_split.split('.')[-2].split('-')[1]), int(filename_split.split('.')[-2].split('-')[0])) 
        self.data_file_end_date = datetime.datetime(int(filename_split.split('.')[-1].split('-')[2]), int(filename_split.split('.')[-1].split('-')[1]), int(filename_split.split('.')[-1].split('-')[0]))

        if 'hour' in self.temporal_resolution:
            if len(self.temporal_resolution) > 4:
                self.n_steps = int(self.temporal_resolution[:-4])
            else:
                self.n_steps = 1
            self.file_format = 'column'
            self.timedelta = relativedelta(hours=self.n_steps)
        elif 'day' in self.temporal_resolution:
            if len(self.temporal_resolution) > 3:
                self.file_format = 'row' 
            else:
                self.file_format = 'column'
                self.timedelta = relativedelta(days=1)
        elif 'week' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'month' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'year' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'var' in self.temporal_resolution:
            self.file_format = 'row'

        self.column_headers = ['start_dt','end_dt','measurement','flag']
        self.parsed_data = {column_header:[] for column_header in self.column_headers}

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter='\t', quotechar='"')
 
            for row_count, row in enumerate(csvreader):

                if self.file_format == 'column':
                    date = row[0]
                    base_dt = datetime.datetime(year=int(date[:4]),month=int(date[5:7]),day=int(date[8:10]),hour=0,minute=0)

                    measurement_inds = np.arange(1,len(row),2,dtype=int)
                    flag_inds = np.arange(2,len(row),2,dtype=int)

                    for step_ii,measurement_ind in enumerate(measurement_inds):
                        self.parsed_data['start_dt'].append(base_dt)
                        self.parsed_data['end_dt'].append(base_dt + self.timedelta)
                        base_dt = copy.deepcopy(self.parsed_data['end_dt'][-1])
                        self.parsed_data['measurement'].append(row[measurement_ind])
                    for flag_ind in flag_inds:
                        self.parsed_data['flag'].append(row[flag_ind])

                elif self.file_format == 'row':
                    start_date = row[0]
                    end_date = row[1]
                    self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[:4]),month=int(start_date[5:7]),day=int(start_date[8:10]),hour=int(start_date[11:13]),minute=int(start_date[14:16])))
                    self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[:4]),month=int(end_date[5:7]),day=int(end_date[8:10]),hour=int(end_date[11:13]),minute=int(end_date[14:16])))
                    self.parsed_data['measurement'].append(row[2])
                    self.parsed_data['flag'].append(row[3])

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)

class ParseEEAAQeReportingFile:

    '''class that handles processing of a EEA AQ eReporting file; new in GHOST 1.6'''

    def __init__(self,filename,run_details):

        self.filename = filename
        self.run_details = run_details
        filename_split = self.filename.split('/')[-1]
        self.component_code = filename_split.split('_')[1]
        #self.measurement_european_group_code = filename_split[12:17]
        self.temporal_resolution = filename_split.split('_')[3][:-4]
        #self.data_file_start_date = datetime.datetime(int(filename_split.split('.')[-2].split('-')[2]), int(filename_split.split('.')[-2].split('-')[1]), int(filename_split.split('.')[-2].split('-')[0])) 
        #self.data_file_end_date = datetime.datetime(int(filename_split.split('.')[-1].split('-')[2]), int(filename_split.split('.')[-1].split('-')[1]), int(filename_split.split('.')[-1].split('-')[0]))

        if 'hour' in self.temporal_resolution:
            if len(self.temporal_resolution) > 4:
                self.n_steps = int(self.temporal_resolution[:-4])
            else:
                self.n_steps = 1
            self.file_format = 'column'
            self.timedelta = relativedelta(hours=self.n_steps)
        elif 'day' in self.temporal_resolution:
            if len(self.temporal_resolution) > 3:
                self.file_format = 'row' 
            else:
                self.file_format = 'column'
                self.timedelta = relativedelta(days=1)
        elif 'week' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'month' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'year' in self.temporal_resolution:
            self.file_format = 'row'
        elif 'var' in self.temporal_resolution:
            self.file_format = 'row'

        self.column_headers = ['start_dt','end_dt','measurement','verification','validity','unit']
        self.parsed_data = {column_header:[] for column_header in self.column_headers}

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')
 
            for row_count, row in enumerate(csvreader):

                # if self.file_format == 'column':
                #     date = row[0]
                #     base_dt = datetime.datetime(year=int(date[:4]),month=int(date[5:7]),day=int(date[8:10]),hour=0,minute=0)

                #     measurement_inds = np.arange(1,len(row),2,dtype=int)
                #     flag_inds = np.arange(2,len(row),2,dtype=int)

                #     for step_ii,measurement_ind in enumerate(measurement_inds):
                #         self.parsed_data['start_dt'].append(base_dt)
                #         self.parsed_data['end_dt'].append(base_dt + self.timedelta)
                #         base_dt = copy.deepcopy(self.parsed_data['end_dt'][-1])
                #         self.parsed_data['measurement'].append(row[measurement_ind])
                #     for flag_ind in flag_inds:
                #         self.parsed_data['flag'].append(row[flag_ind])

                #if self.file_format == 'row':
                if row_count == 0:
                    continue # header
                else:
                    start_date = row[2]
                    end_date = row[3]
                    #print(self.filename, start_date)
                    if len(start_date) > 11: # is there time included in date?
                        self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[:4]),month=int(start_date[5:7]),day=int(start_date[8:10]),hour=int(start_date[11:13]),minute=int(start_date[14:16])))
                        self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[:4]),month=int(end_date[5:7]),day=int(end_date[8:10]),hour=int(end_date[11:13]),minute=int(end_date[14:16])))
                    else: 
                        #print(self.filename, 'success')
                        self.parsed_data['start_dt'].append(datetime.datetime(year=int(start_date[:4]),month=int(start_date[5:7]),day=int(start_date[8:10])))
                        self.parsed_data['end_dt'].append(datetime.datetime(year=int(end_date[:4]),month=int(end_date[5:7]),day=int(end_date[8:10])))
                    #print(self.filename, self.parsed_data['start_dt'])
                    if self.run_details['parameter'] == 'H+_PREC':
                        if float(row[4]) > 0:
                            self.parsed_data['measurement'].append(str(10**-float(row[4]))) # convert parsed pH values to H+ concentration in mol l-1
                        else:
                            self.parsed_data['measurement'].append('') # convert parsed pH values to H+ concentration in mol l-1
                        self.parsed_data['unit'].append('mol l-1') # parse artificially H+ unit
                    else:
                        self.parsed_data['measurement'].append(row[4])
                        self.parsed_data['unit'].append(row[5].replace('s.', 'S').replace('n.', 'N').replace('.', '')) # capitalise s and n to make sure unit converter understands sulpher and nitrogen

                    self.parsed_data['verification'].append(row[8])
                    self.parsed_data['validity'].append(row[7])

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 

        #print(self.filename, self.parsed_data['start_dt'])
        return self.parsed_data


class ParseINDAAFFile:

    '''class that handles processing of a INDAAF precip file'''

    def __init__(self,filename, run_details, parameter_details):

        self.filename = filename
        self.parameter_details = parameter_details
        self.run_details = run_details
        self.parsed_metadata = {}
        self.to_correct = {'Ca2+':'Ca++_PREC','Mg2+':'Mg++_PREC','K+':'K+_PREC','Na+':'Na+_PREC','SO4--':'SO4--_PREC', 'Cl-':'Cl-_PREC', 'NO3-':'NO3-_PREC', 'NH4+':'NH4+_PREC','H': 'PREC_ACCUM', 'H+': 'H+_PREC'}
        self.to_correct_pm = {'Ca2+':['Ca++','PM10_Ca++','PM2.5_Ca++'],'Mg2+':['Mg++','PM10_Mg++','PM2.5_Mg++'],'K+':['K+','PM10_K+','PM2.5_K+'],'Na+':['Na+','PM10_Na+','PM2.5_Na+'],'SO4--':['SO4--','PM10_SO4--','PM2.5_SO4--'], 'Cl-':['Cl-','PM10_Cl-','PM2.5_Cl-'], 'NO3-':['NO3-','PM10_NO3-','PM2.5_NO3-'], 'NH4+':['NH4+','PM10_NH4+','PM2.5_NH4+'],'TC':['C','PM10_C','PM2.5_C'], 'OC':['OC','PM10_OC','PM2.5_OC'], 'EC':['EC','PM10_EC','PM2.5_EC'], 'Al-27[#2]':['Al','PM10_Al','PM2.5_Al'], 'Fe-56[#2]':['Fe','PM10_Fe','PM2.5_Fe'],'Mn-55[#2]':['Mn','PM10_Mn','PM2.5_Mn'],'Pb-208[#3]':['Pb','PM10_Pb','PM2.5_Pb'],'Cr-52[#2]':['Cr','PM10_Cr','PM2.5_Cr'],'Cu-63[#2]':['Cu','PM10_Cu','PM2.5_Cu'],'Zn-66[#2]':['Zn','PM10_Zn','PM2.5_Zn'], 'Cd-111[#3]':['Cd','PM10_Cd','PM2.5_Cd'],'Ni-60[#2]':['Ni','PM10_Ni','PM2.5_Ni']}

        # needed to map the column headers to the processed component GHOST name for all pm types
        if 'PM10' in self.run_details['parameter']:
            self.pm_matrix = 1
        elif 'PM2.5' in self.run_details['parameter']:
            self.pm_matrix = 2
        else: # PM
            self.pm_matrix = 0


    def read_data(self):

        if ('PREC' in self.run_details['parameter']) or (self.parameter_details['matrix'] in ['gas', 'pm10', 'pm2.5', 'pm']):
            delim = '\t'
        else:
            delim = ' '
        
        with open(self.filename, newline='', encoding="ISO-8859-16") as csvfile:
            csvreader = csv.reader(csvfile, delimiter=delim, quotechar="'")

            # count lines of header
            start_data = False
            parse_meta_table = False

            for row_count, row in enumerate(csvreader):

                if ('PREC' in self.run_details['parameter']) or (self.parameter_details['matrix'] == 'gas'): # prec files (excluding precip amount files) and gas files
                    
                    # parse lod, lod unit, units etc.
                    if '#variable' in row[0]:
                        #for to_c in self.to_correct.keys():
                        bracket_component = '('+list(self.to_correct.keys())[list(self.to_correct.values()).index(self.run_details['parameter'])]+')'

                        if bracket_component in row[0]: # choose right parameter
                            meta_strs = row[0].split(',')
                            if 'H+_PREC' in self.run_details['parameter']:
                                self.parsed_metadata['unit'] = 'ueq/l'
                            else:
                                self.parsed_metadata['unit'] = meta_strs[1].strip()
                                
                            for meta_str in meta_strs:
                                if 'Detection limit' in meta_str:
                                    self.parsed_metadata['lod'] = meta_str.split('=')[-1].split(' ')[0]
                                    self.parsed_metadata['lod unit'] = meta_str.split('=')[-1].split(' ')[-1]
                                if 'Instrument type' in meta_str:
                                    self.parsed_metadata['Instrument type rainfall'] = meta_str.split('=')[-1]
                                if 'Analytical measurement technique' in meta_str:
                                    self.parsed_metadata['Measurement technique rainfall'] = meta_str.split('=')[-1]
                                if 'Analytical instrument manufacturer' in meta_str:
                                    self.parsed_metadata['Instrument manufacturer rainfall'] = meta_str.split('=')[-1]
                                if 'Analytical instrument model' in meta_str:
                                    self.parsed_metadata['Instrument rainfall'] = meta_str.split('=')[-1]

                    # parse other meta variables
                    if '#sample duration' in row[0].lower():
                        self.parsed_metadata['Time resolution'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station code' in row[0].lower():
                        self.parsed_metadata['Station code'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    elif '#station:' in row[0].lower():
                        self.parsed_metadata['Station code'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#country' in row[0].lower():
                        self.parsed_metadata['Country'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station land use' in row[0].lower():
                        self.parsed_metadata['Land use'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station setting' in row[0].lower():
                        self.parsed_metadata['Station setting'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station latitude' in row[0].lower():
                        self.parsed_metadata['Latitude'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station longitude' in row[0].lower():
                        self.parsed_metadata['Longitude'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station altitude' in row[0].lower():
                        self.parsed_metadata['Altitude'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1][:-1]
                    if '#measurement height' in row[0].lower():
                        self.parsed_metadata['Measurement height'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1][:-1]
                    if '#analytical laboratory code' in row[0].lower():
                        self.parsed_metadata['Laboratory Code'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#instrument type' in row[0].lower():
                        self.parsed_metadata['Instrument type'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#analytical measurement technique' in row[0].lower():
                        self.parsed_metadata['Measurement technique'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#analytical instrument manufacturer' in row[0].lower():
                        self.parsed_metadata['Instrument manufacturer'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#analytical instrument model' in row[0].lower():
                        self.parsed_metadata['Instrument'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1].replace('and', ',')
                    if '#DOI' in row[0]:
                        #row = [x for x in row if x]
                        self.parsed_metadata['Sampling point identifier'] = row[0][-12:]

                    if start_data:
                        # parse measurements
                        for column_header in column_headers:
                            if self.run_details['parameter'] == column_header:
                                meas = row[column_headers.index(column_header)]
                                self.parsed_data[column_header].append(meas)

                        # parse dates
                        if (row[0] != ''):
                            split_date = row[0].split(' ') # split in date and time

                            # date
                            date = split_date[0].replace('/', '-').split('-')
                            if len(date[0]) == 4: # format YYYY-MM-DD
                                start_year = date[0]
                                start_month = date[1]
                                if len(date) > 2: # day also given
                                    start_day = date[2]
                                else:
                                    start_day = 1
                            else: # format DD-MM-YYYY
                                start_year = date[2]
                                start_month = date[1]
                                start_day = date[0]

                            #time
                            if len(split_date) > 1: # time also given
                                time = split_date[1].split(':')
                                start_hour = time[0]
                                start_minute = time[1]
                            else:
                                start_hour = 0
                                start_minute = 0

                            start_full_date = datetime.datetime(year=int(start_year),month=int(start_month),day=int(start_day),hour=int(start_hour),minute=int(start_minute))
                            self.parsed_data['start_dt'].append(start_full_date)

                            if self.parsed_metadata['Time resolution'] == '1d':
                                self.parsed_data['end_dt'].append(start_full_date + relativedelta(days=1))
                            elif self.parsed_metadata['Time resolution'] == '1month':
                                self.parsed_data['end_dt'].append(start_full_date + relativedelta(months=1))
                            else:
                                print('Time resolution in network parser not coded')

                    if 'Date' in row[0]: # start of data
                        self.parsed_data = {'start_dt':[], 'end_dt':[], self.run_details['parameter']:[]}
                        start_data=True
                        header = row

                        # find header and correct names
                        column_headers = [element.replace(' ', '') for element in header]
                        for to_c in self.to_correct.keys():
                            if to_c in column_headers:
                                column_headers[column_headers.index(to_c)] = self.to_correct[to_c]

                elif self.run_details['parameter'] == 'PM10': # PM10 concentration files
                    if start_data:
                        # parse dates
                        if (row[0] != ''):
                            date = row[0].split('-')
                            start_day = date[2]
                            start_month = date[1]
                            start_year = date[0]

                            time = row[1].split(':')
                            start_hour = time[0]
                            start_minute = time[1]

                            start_full_date = datetime.datetime(year=int(start_year),month=int(start_month),day=int(start_day),hour=int(start_hour),minute=int(start_minute))
                            self.parsed_data['start_dt'].append(start_full_date)
                            self.parsed_data['end_dt'].append(start_full_date + relativedelta(hours=1)) # time resolution of one hour

                            #measurements
                            if row[2] == 'NA':
                                self.parsed_data[self.run_details['parameter']].append('')
                            else:
                                self.parsed_data[self.run_details['parameter']].append(row[2])


                    # parse metadata header
                    if '#latitude' in row[0].lower():
                        self.parsed_metadata['Latitude'] = row[2].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#longitude' in row[0].lower():
                        self.parsed_metadata['Longitude'] = row[2].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station' in row[0].lower():
                        self.parsed_metadata['Station code'] = row[2].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#date' in row[0].lower():
                        self.parsed_metadata['unit'] = row[2].split('(')[1].replace(')', '')
                    if row[0].lower() == '#data':
                        self.parsed_metadata['Data level'] = row[2]
                    if '#PM10_' in row[0]:
                        row = [x for x in row if x]
                        self.parsed_metadata['Sampling point identifier'] = row[-1][-12:]
                    # parse data
                    if 'Date' in row[0]: # start of data
                        self.parsed_data = {'start_dt':[], 'end_dt':[], self.run_details['parameter']:[]}
                        start_data=True
 
                elif self.parameter_details['matrix'] in ['pm', 'pm2.5', 'pm10']: # aerocol composition 

                    if self.run_details['parameter'] in ['PM10_Al','PM10_Fe','PM10_Mn','PM10_Pb','PM10_Cr','PM10_Cu','PM10_Zn','PM10_Cd','PM10_Ni', 'PM2.5_Al','PM2.5_Fe','PM2.5_Mn','PM2.5_Pb','PM2.5_Cr','PM2.5_Cu','PM2.5_Zn','PM2.5_Cd','PM2.5_Ni','Al','Fe','Mn','Pb','Cr','Cu','Zn','Cd','Ni']:
                        indaaf_matrix = 'Tracemetals'
                    elif self.run_details['parameter'] in ['PM10_C','PM10_EC','PM10_OC','PM2.5_C','PM2.5_EC', 'PM2.5_OC','C', 'EC', 'OC']:
                        indaaf_matrix = 'ECOCTC'
                    else:
                        indaaf_matrix = 'InorganicsandOrganics'

                    if len(row) < 1: # skip empty lines
                        continue
                    
                    # parse meta variables
                    if ('#sampling=' in row[0].lower()) and ('period' not in row[0].lower()):
                        self.parsed_metadata['Time resolution'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station code' in row[0].lower():
                        self.parsed_metadata['Station code'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    elif '#station:' in row[0].lower():
                        self.parsed_metadata['Station code'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#country' in row[0].lower():
                        self.parsed_metadata['Country'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station land use' in row[0].lower():
                        self.parsed_metadata['Land use'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station setting' in row[0].lower():
                        self.parsed_metadata['Station setting'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station latitude' in row[0].lower():
                        self.parsed_metadata['Latitude'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station longitude' in row[0].lower():
                        self.parsed_metadata['Longitude'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                    if '#station altitude' in row[0].lower():
                        self.parsed_metadata['Altitude'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1][:-1]
                    if '#measurement height' in row[0].lower():
                        self.parsed_metadata['Measurement height'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1][:-1]
                    if '#DOI' in row[0]:
                        self.parsed_metadata['Sampling point identifier'] = row[0][-12:]

                    if '#component' in row[0].lower():
                        if row[0].replace(' ', '').replace('=', ':').split(':')[-1].strip() == indaaf_matrix:
                            parse_meta_table = True
                        else:
                            parse_meta_table = False

                    if parse_meta_table:
                        if '#analytical laboratory code' in row[0].lower():
                            self.parsed_metadata['Laboratory Code'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                        if '#instrument type' in row[0].lower():
                            self.parsed_metadata['Instrument type'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                        if '#analytical measurement technique' in row[0].lower():
                            self.parsed_metadata['Measurement technique'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                        if '#analytical instrument manufacturer' in row[0].lower():
                            self.parsed_metadata['Instrument manufacturer'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                        if '#analytical instrument model' in row[0].lower():
                            self.parsed_metadata['Instrument'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1].replace('and', ',')
                        elif '#analytical instrument serial number' in row[0].lower():
                            self.parsed_metadata['Instrument'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1].replace('and', ',')
                        elif '#instrument name' in row[0].lower():
                            self.parsed_metadata['Instrument'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1].replace('and', ',')
                        if '#inlet description' in row[0].lower():
                            self.parsed_metadata['Inlet description'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                        if '#flow rate' in row[0].lower():
                            self.parsed_metadata['Flow rate'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                        if '#exposed filter area' in row[0].lower():
                            self.parsed_metadata['Exposed filter area'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]
                        if '#medium' in row[0].lower():
                            self.parsed_metadata['Medium'] = row[0].replace(' ', '').replace('=', ':').split(':')[-1]

                        if '#variable' in row[0]:
                            # find index in mapping dict that fits componenent
                            for i, indaaf_component in enumerate(self.to_correct_pm):
                                if self.run_details['parameter'] in self.to_correct_pm[indaaf_component]:
                                    index = i
                                    break
                            bracket_component = '('+list(self.to_correct_pm.keys())[index]+')'
                            if indaaf_matrix == 'ECOCTC':
                                bracket_component = bracket_component.replace('(', '').replace(')', '')

                            # parse unit
                            if bracket_component in row[0]: # choose right parameter
                                meta_strs = row[0].split(',')
                                # general case
                                self.parsed_metadata['unit'] = meta_strs[1].strip()

                        # metals extra case for unit parsing
                        if self.run_details['parameter'] in ['PM10_Al','PM10_Fe','PM10_Mn','PM10_Pb','PM10_Cr','PM10_Cu','PM10_Zn','PM10_Cd','PM10_Ni', 'PM2.5_Al','PM2.5_Fe','PM2.5_Mn','PM2.5_Pb','PM2.5_Cr','PM2.5_Cu','PM2.5_Zn','PM2.5_Cd','PM2.5_Ni','Al','Fe','Mn','Pb','Cr','Cu','Zn','Cd','Ni']:
                            self.parsed_metadata['unit'] = 'ng/m3'

                    if start_data:
                        # parse measurements
                        for column_header in column_headers:
                            if self.run_details['parameter'] == column_header:
                                meas = row[column_headers.index(column_header)]
                                self.parsed_data[column_header].append(meas)

                        # parse dates
                        if (row[0] != ''):
                            split_date = row[0].split(' ') # split in date and time

                            # date
                            date = split_date[0].replace('/', '-').split('-')
                            if len(date[0]) == 4: # format YYYY-MM-DD
                                start_year = date[0]
                                start_month = date[1]
                                if len(date) > 2: # day also given
                                    start_day = date[2]
                                else:
                                    start_day = 1
                            else: # format DD-MM-YYYY
                                start_year = date[2]
                                start_month = date[1]
                                start_day = date[0]

                            #correct day
                            if (start_year == '2001') and (start_month == '02') and (start_day == '29'):
                                start_day = '28'

                            #time
                            if len(split_date) > 1: # time also given
                                time = split_date[1].split(':')
                                start_hour = time[0]
                                start_minute = time[1]
                            else:
                                start_hour = 0
                                start_minute = 0

                            start_full_date = datetime.datetime(year=int(start_year),month=int(start_month),day=int(start_day),hour=int(start_hour),minute=int(start_minute))
                            self.parsed_data['start_dt'].append(start_full_date)

                            if (self.parsed_metadata['Time resolution'] == '1d') or ('1day' in self.parsed_metadata['Time resolution']):
                                self.parsed_data['end_dt'].append(start_full_date + relativedelta(days=1))
                            elif self.parsed_metadata['Time resolution'] == '1month':
                                self.parsed_data['end_dt'].append(start_full_date + relativedelta(months=1))
                            elif 'weeklysamples' in self.parsed_metadata['Time resolution']:
                                self.parsed_data['end_dt'].append(start_full_date + relativedelta(weeks=1))
                            else:
                                print('Time resolution in network parser not coded: {}'.format(self.parsed_metadata['Time resolution']))

                    if (row[0].strip().replace(' ', '_') == 'Start_datetime') or (row[0].strip() == 'Date'): # start of data
                        self.parsed_data = {'start_dt':[], 'end_dt':[], self.run_details['parameter']:[]}
                        start_data=True
                        header = row

                        # find header and correct names
                        column_headers = [element.replace(' ', '') for element in header]
                        for to_c in self.to_correct_pm.keys():
                            if to_c in column_headers:
                                column_headers[column_headers.index(to_c)] = self.to_correct_pm[to_c][self.pm_matrix]

                    
        # check if lengths of data arrays are the same
        if len(self.parsed_data['start_dt']) < len(self.parsed_data[self.run_details['parameter']]):
            self.parsed_data[self.run_details['parameter']] = self.parsed_data[self.run_details['parameter']][:len(self.parsed_data['start_dt'])]

        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)

        return self.parsed_metadata


class ParseJAPANNIESFile:

    '''class that handles processing of a JAPAN NIES file'''

    def __init__(self,filename,file_type):

        self.filename = filename
        self.file_type = file_type
        
        if self.file_type == 'hourly':
            self.file_type = 'hourly'
            self.column_dict = {'year':0,'station_reference':1,'species':3,'units':4,'month':5,'day':6,'hour1':7}
            self.column_headers = ['station_reference','species','units','start_dt','end_dt','measurement']
        elif self.file_type == 'monthly': 
            self.file_type = 'monthly'
            self.column_dict = {'year':0,'species':3,'station_reference':11,'measured_days':35,'measured_hours':47,'april':59}
            self.column_headers = ['station_reference','species','units','start_dt','end_dt','measured_hours_percent','measurement']
            self.units = {'SO2':'PPM','NO':'PPM','NO2':'PPM','CO':'PPM','OX':'PPM','NMHC':'PPMC','CH4':'PPMC','THC':'PPMC','SPM':'MG/M3','PM25':'UG/M3'}


        self.parsed_data = {column_header:[] for column_header in self.column_headers}

    def read_data(self):

        with open(self.filename, newline='', encoding='cp932') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',',quotechar='"') 
            for row_count, row in enumerate(csvreader):
                
                if row_count > 0:

                    #strip row
                    row = [x.strip() for x in row]

                    #hourly files
                    if self.file_type == 'hourly':
                        
                        #first check if current day exists 
                        try:
                            dt_day = datetime.datetime(year=int(row[self.column_dict['year']]), month=int(row[self.column_dict['month']]), day=int(row[self.column_dict['day']]), hour=0, minute=0)
                        #if not then move to next line
                        except ValueError:
                            continue
                    
                        #iterate through hours
                        for hh in range(24):
                            #time
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(row[self.column_dict['year']]), month=int(row[self.column_dict['month']]), day=int(row[self.column_dict['day']]), hour=hh, minute=0))
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                            
                            #measurements
                            meas = row[self.column_dict['hour1']+hh]
                            #set measurement fill values (9997,9998,9999,'') as NaN values
                            if meas in ['9997','9998','9999','']:
                                self.parsed_data['measurement'].append(np.NaN)
                            else:
                                self.parsed_data['measurement'].append(meas)
                    
                            #process other variables
                            for var in ['station_reference','species','units']:
                                self.parsed_data[var].append(row[self.column_dict[var]])

                    #monthly files
                    elif self.file_type == 'monthly':

                        #iterate through months
                        for mm_ii, mm in enumerate([4,5,6,7,8,9,10,11,12,1,2,3]):
                            #time
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(row[self.column_dict['year']]), month=mm, day=1, hour=0, minute=0))
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(months=1))

                            #process misc. variables
                            for var in ['station_reference','species']:
                                self.parsed_data[var].append(row[self.column_dict[var]])

                            #add units --> not in file but add information from metadata file
                            self.parsed_data['units'].append(self.units[self.parsed_data['species'][-1]])

                            #measurements
                            if self.parsed_data['species'][-1] in ['NMHC','CH4','THC','PM25']:
                                meas = row[self.column_dict['measured_hours']+mm_ii]
                            else:
                                meas = row[self.column_dict['april']+mm_ii]
                            #set measurement fill values (9997,9998,9999,'') as NaN values
                            if meas in ['9997','9998','9999','']:
                                self.parsed_data['measurement'].append(np.NaN)
                            else:
                                self.parsed_data['measurement'].append(meas)

                            #measured hours --> calculate percent in month
                            if self.parsed_data['species'][-1] in ['NMHC','CH4','THC','PM25']:
                                measured_hours = row[self.column_dict['measured_days']+mm_ii]
                                if (measured_hours != '') & (self.parsed_data['species'][-1] == 'PM25'):
                                    measured_hours = float(measured_hours) * 24.0
                            else:
                                measured_hours = row[self.column_dict['measured_hours']+mm_ii]

                            if measured_hours == '':
                                measured_hours_percent = ''
                            else:
                                measured_hours = float(measured_hours)
                                n_hours_in_month = monthrange(int(row[self.column_dict['year']]),mm)[1]*24.0
                                measured_hours_percent = (100.0/n_hours_in_month)* measured_hours
                            self.parsed_data['measured_hours_percent'].append(measured_hours_percent)

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                elif column_header == 'measurement':
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.float32)
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)                  


class ParseMEXICOCDMXFile:

    '''class that handles processing of a MEXICO CDMX file'''

    def __init__(self,filename):

        self.filename = filename
        self.start_read = False

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row[0] == 'date':
                    column_headers = copy.deepcopy(row)
                    headers_to_alter = {'cve_station':'id_station','cve_parameter':'id_parameter','date':'start_dt'}
                    column_headers = [headers_to_alter[column_header] if column_header in list(headers_to_alter.keys()) else column_header for column_header in column_headers]
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                    self.parsed_data['end_dt'] = []
                    self.start_read = True
                #data lines
                elif self.start_read:
                    for column_header, element in zip(column_headers, row):
                        if column_header == 'start_dt':
                            self.parsed_data['start_dt'].append(datetime.datetime(year=int(element[6:10]),month=int(element[3:5]),day=int(element[:2]),hour=int(element[11:13])-1,minute=int(element[14:16]))) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                        else:
                            self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 


class ParseMITECOFile:

    '''class that handles processing of a MITECO file'''

    def __init__(self,filename,parameter_details):

        #add passed variables to self
        self.filename = filename
        self.parameter_details = parameter_details

        #get file type
        if '_HH_' in self.filename:
            self.file_type = 'hourly'
        elif '_DD_' in self.filename:
            self.file_type = 'daily'
        elif '_Irreg_' in self.filename:
            self.file_type = 'irregular'

        #initialise fields to store data columns
        self.punto_muestreo = []
        self.start_dt = []
        self.temporal_resolutions = []
        self.measurements = []

    def read_data(self):

        with open(self.filename) as f:
            self.content = f.readlines()
            self.content = [x.strip('\n') for x in self.content]

            column_headers = self.content[0].split(';') 

            for row in self.content[1:]:
                data_line = row.split(';')

                #if species code on line is not correct for species processing, move to next line
                if self.parameter_details['miteco_code'] != data_line[column_headers.index('MAGNITUD')]:
                    continue             

                #get time/temporal resolution/measurement
                if self.file_type == 'hourly':
                    n_valid_measurements = 0
                    first_measurement_index = column_headers.index('H01')
                    for hour in range(24):
                        try:
                            dt = datetime.datetime(int(data_line[column_headers.index('ANNO')]),int(data_line[column_headers.index('MES')]),int(data_line[column_headers.index('DIA')]),hour,0)
                            meas = data_line[first_measurement_index+hour] 
                            self.start_dt.append(dt)
                            self.temporal_resolutions.append(60)    
                            self.measurements.append(meas)
                            n_valid_measurements+=1
                        except (ValueError, IndexError) as err:
                            continue

                elif self.file_type == 'daily':
                    n_valid_measurements = 0
                    first_measurement_index = column_headers.index('D01')
                    for day in range(1,32):
                        try:
                            dt = datetime.datetime(int(data_line[column_headers.index('ANNO')]),int(data_line[column_headers.index('MES')]),day,0,0)
                            meas = data_line[first_measurement_index+(day-1)]
                            self.start_dt.append(dt)
                            self.temporal_resolutions.append(60*24)
                            self.measurements.append(meas)
                            n_valid_measurements+=1
                        except (ValueError, IndexError) as err:  
                            continue

                elif self.file_type == 'irregular':  
                    n_valid_measurements = 0
                    try:
                        start_date = data_line[column_headers.index('FECHA_INICIAL')]
                        end_date = data_line[column_headers.index('FECHA_FINAL')]
                        if len(start_date) == 19:
                            start_yyyy = int(start_date[:4])
                            start_mm = int(start_date[5:7])
                            start_dd = int(start_date[8:10])
                            start_hour = int(start_date[11:13]) 
                            start_min = int(start_date[14:16])
                            start_sec = int(start_date[17:19])
                        elif len(start_date) == 10:
                            start_yyyy = int(start_date[:4])
                            start_mm = int(start_date[5:7])
                            start_dd = int(start_date[8:10])
                            start_hour = 0 
                            start_min = 0
                            start_sec = 0
                        else:
                            1+'a'

                        if len(end_date) == 19:
                            end_yyyy = int(end_date[:4])
                            end_mm = int(end_date[5:7])
                            end_dd = int(end_date[8:10])
                            end_hour = int(end_date[11:13]) 
                            end_min = int(end_date[14:16])
                            end_sec = int(end_date[17:19])
                        elif len(end_date) == 10:
                            end_yyyy = int(end_date[:4])
                            end_mm = int(end_date[5:7])
                            end_dd = int(end_date[8:10])
                            end_hour = 0 
                            end_min = 0
                            end_sec = 0
                        else:
                            1+'a'

                        start_dt = datetime.datetime(start_yyyy,start_mm,start_dd,start_hour,start_min,start_sec)
                        end_dt = datetime.datetime(end_yyyy,end_mm,end_dd,end_hour,end_min,end_sec)
                        dt_window = end_dt - start_dt
                        res = int(dt_window.total_seconds()/60.)
                        meas = data_line[column_headers.index('VALOR')]
                        self.start_dt.append(start_dt)
                        self.temporal_resolutions.append(res)
                        self.measurements.append(meas)
                        n_valid_measurements = 1
                    
                    except IndexError:
                        pass

                #if have no valid measurements, move to next line
                if n_valid_measurements == 0:
                    continue
                    
                #get punto_muestreo
                self.punto_muestreo.extend([data_line[column_headers.index('PUNTO_MUESTREO')]]*n_valid_measurements)

        #convert lists to numpy arrays
        self.punto_muestreo = np.array(self.punto_muestreo, dtype=np.str_)
        self.start_dt = np.array(self.start_dt)
        self.temporal_resolutions = np.array(self.temporal_resolutions, dtype=np.int32)
        self.measurements = np.array(self.measurements, dtype=np.str_)

class ParseNASAAmesFile:

    '''class that handles processing of a nasa ames file'''

    def __init__(self,filename,station_reference,run_details):
        
        self.filename = filename
        self.station_reference = station_reference
        self.run_details = run_details

        #core header meta
        self.n_header = ''          #number of header lines
        self.ffi = ''               #file format index
                                    #1st digit indicates the number of independent variables (dimensions)
                                    #2nd digit indicates number of necessary auxiliary variable
                                    #Remaining differences cover the way the independent variables increment and the nature (numeric or character) of the variables
        self.originator_name = ''   #data originator name
        self.originator_affil = ''  #data originator affiliation
        self.provider_name = ''     #data provider name
        self.campaign_name = ''     #campaign name
        self.volume_number = ''     #current volume number (dated field - should be 1)
        self.n_volumes = ''         #number of volumes in dataset (dated field - should be 1)
        self.date_start = ''        #date for which record starts
        self.file_creation = ''     #date of file creation


        #independent variable meta (e.g. time)
        self.independent_interval = ''  #independent variable interval (if changes should be zero)
        self.independent_name = ''      #independent variable name (e.g. days from file reference point)


        #primary variable meta (quantities that have been observed or derived)
        self.n_primary = ''               #number of primary variables
        self.primary_scale_factors = ''   #scale factors for each of primary variables
        self.primary_missing_values = ''  #value imposed for missing data for each primary variable
        self.primary_names = []           #primary variable names (e.g. ozone)
        self.primary_units = []           #primary variable units
        self.primary_details = []         #extra details about variables (e.g. limit of detection)

        #auxiliary variable meta
        self.n_auxiliary = ''    #number of auxiliary variables

        #comments meta
        self.n_comments = ''     #number of lines of comments
        self.comments = []       #extra meta comments about data. Can also include column headings

        #important line counts
        self.endline_primary_count = -999
        self.start_auxiliary = -999
        self.start_comments = -999
        self.start_data = -999
        self.row_count = 0

    def read_header(self):

        with open(self.filename) as f:
            self.content = f.readlines()
            self.content = [x.strip('\n') for x in self.content]

            for row in self.content:

                #--read core header meta--
                if self.row_count == 0:
                    line_split = row.split(' ')
                    self.n_header,self.ffi = line_split[0],line_split[1]

                elif self.row_count == 1:
                    self.originator_name = row

                elif self.row_count == 2:
                    self.originator_affil = row

                elif self.row_count == 3:
                    self.provider_name = row

                elif self.row_count == 4:
                    self.campaign_name = row

                elif self.row_count == 5:
                    line_split = row.split(' ')
                    self.volume_name, self.n_volumes = line_split[0],line_split[1]

                elif self.row_count == 6:
                    line_split = row.split(' ')
                    self.date_start, self.file_creation = line_split[0]+line_split[1]+line_split[2], line_split[3]+line_split[4]+line_split[5]

                #--read independent variable meta--
                elif self.row_count == 7:
                    self.independent_interval = row

                elif self.row_count == 8:
                    self.independent_name = row

                #--read primary variable meta--
                elif self.row_count == 9:
                    self.n_primary = int(row)
                    self.endline_primary_count = 11+self.n_primary
                    self.start_auxiliary = self.endline_primary_count+1

                elif self.row_count == 10:
                    self.primary_scale_factors = row.split(' ')

                elif self.row_count == 11:
                    self.primary_missing_values = row.split(' ')

                elif (self.row_count > 11) & (self.row_count <= self.endline_primary_count):
                    #handle line splitting with quotes (with commas inside)
                    line_split = []
                    split_by_quotes = row.split('"')
                    for split_ii, split_element in enumerate(split_by_quotes):
                        if (split_ii % 2) == 0:
                            line_split = np.append(line_split, split_element.split(','))
                        else:
                            if split_element[:7] == 'Comment':
                                new_str = 'Comment="{}"'.format(split_element[8:])
                            else: 
                                new_str = '{}"{}"'.format(line_split[-1], split_element)
                                line_split = line_split[:-1]
                            line_split = np.append(line_split, new_str)
                    line_split = [element.strip() for element in line_split]
                    line_split = [element for element in line_split if element != '']
                    if len(line_split) == 0:
                        line_split = [row]

                    #only read line if first value is one of accepted primary variable names
                    #otherwise line is appended to primary details - this deals with badly formatted files
                    #also add 1 to auxiliary start count - so don't start reading too soon

                    #create a list of all potentially valid primary variables
                    valid_primary_variables = ['end_time of measurement','numflag']
                    for param in list(standard_parameters.keys()):
                        ebas_parameter_names = standard_parameters[param]['ebas_parameter_name']
                        for ebas_parameter_name in ebas_parameter_names:
                            #only process variables with a valid EBAS parameter name (i.e. not empty string)
                            if ebas_parameter_name != '':
                                valid_primary_variables=np.append(valid_primary_variables,ebas_parameter_name)
                                #create numflag variable for all parameters
                                valid_primary_variables=np.append(valid_primary_variables,'numflag {}'.format(ebas_parameter_name))                    

                    if (line_split[0] in valid_primary_variables) or (line_split[0] in ['precipitation_amount_off','numflag precipitation_amount_off']):
                        self.primary_names.append(line_split[0])
                        self.primary_units.append(line_split[1])
                        if len(line_split) > 2:
                            self.primary_details.append(line_split[2:])
                        else:
                            self.primary_details.append([])
                    else:
                        self.primary_details[-1] = self.primary_details[-1]+line_split
                        self.endline_primary_count+=1
                        self.start_auxiliary+=1

                #--read auxiliary variable meta--
                #first checks if row is a valid number or relates to the previous primary details not correctly formatted
                #if so then append to primary details
                elif self.row_count == self.start_auxiliary:
                    try:
                        self.n_auxiliary = int(row)
                        #if is not 0 then break processing and panic - as have never seen this be used and don't know format
                        if self.n_auxiliary != 0:
                            self.run_details['log_file_str'] += 'Using auxilary variables. {} of them! How Strange. Now need to code for this. In file: {} .Stopping processing.'.format(self.n_auxiliary, self.filename)
                            create_GHOST_stage_logfile(2,self.run_details)
                        self.start_comments = self.start_auxiliary+1

                    except:
                        line_split = row.split(',')
                        self.primary_details[-1] = np.append(self.primary_details[-1],line_split)
                        self.endline_primary_count+=1
                        self.start_auxiliary+=1

                #--read comments meta--
                elif self.row_count == self.start_comments:
                    self.n_comments = int(row) # -999
                    self.end_comments = self.start_comments+self.n_comments
                    self.start_data = self.end_comments+1

                elif (self.row_count > self.start_comments) & (self.row_count <= self.end_comments):
                    self.comments.append(row)

                #manually iterate row count
                self.row_count+=1

            if self.start_data == -999: # ??????????????
                self.start_data = int(self.n_header)
            

    def read_data(self):

        #cut file read to just be data columns
        data = self.content[self.start_data:]

        #make list of lists to store data columns (1 independent variable + primary variables)
        self.structured_data = [[] for _ in range(self.n_primary+1)]

        #determine if any multiplication by scale factors is necessary
        primary_scale_factor_conversion = [True if float(sf) != 1.0 else False for sf in self.primary_scale_factors]

        #iterate through lines, appending column data into a list of lists
        for line in data:
            line_split = line.split()
            for n,var in enumerate(line_split):
                #not independent data variable?
                if n > 0:
                    #convert missing values to NaNs if necessary
                    if var == self.primary_missing_values[n-1]:
                        var = np.NaN
                    #do multiplication by scale factors if necessary (if non NaN)
                    if primary_scale_factor_conversion[n-1]:
                        try:
                            var = float(var)
                        except:
                            var = np.NaN
                        if np.isfinite(var):
                            var = var * float(self.primary_scale_factors[n-1])

                    # convert pH to H+ concentration
                    if self.run_details['parameter'] == 'H+_PREC':
                        if n == self.n_primary-1:
                            var = 10**(-float(var))
                #append
                self.structured_data[n].append(var)


"""A dictionary, where key is the ISD identifier in ASCII and value
is a list that contains [variable_name, number_of_chars]. The variable
name is as in ISD format. The number of chars indicate the number of
characters data (without considering the 3 chars of the identifier"""


isd_map = {'AA1': ['LIQUID-PRECIPITATION', 8],
           'AA2': ['LIQUID-PRECIPITATION', 8],
           'AA3': ['LIQUID-PRECIPITATION', 8],
           'AA4': ['LIQUID-PRECIPITATION', 8],
           'AB1': ['LIQUID-PRECIPITATION MONTHLY TOTAL', 7],
           'AC1': ['PRECIPITATION-OBSERVATION-HISTORY', 3],
           'AD1': ['LIQUID-PRECIPITATION GREATEST AMOUNT IN 24 HOURS, FOR THE MONTH', 19],
           'AE1': ['LIQUID-PRECIPITATION, NUMBER OF DAYS WITH SPECIFIC AMOUNTS, FOR THE MONTH', 12],
           'AG1': ['PRECIPITATION-ESTIMATED-OBSERVATION', 4],
           'AH1': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH2': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH3': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH4': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH5': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AH6': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI1': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI2': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI3': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI4': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI5': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AI6': ['LIQUID-PRECIPITATION MAXIMUM SHORT DURATION', 15],
           'AJ1': ['SNOW-DEPTH', 14],
           'AK1': ['SNOW-DEPTH-GREATEST-ON-GROUND', 12],
           'AL1': ['SNOW-ACCUMULATION', 7],
           'AL2': ['SNOW-ACCUMULATION', 7],
           'AL3': ['SNOW-ACCUMULATION', 7],
           'AL4': ['SNOW-ACCUMULATION', 7],
           'AM1': ['SNOW-ACCUMULATION-GREATEST-IN-24H', 18],
           'AN1': ['SNOW-ACCUMULATION-FOR-MONTH', 9],
           'AO1': ['LIQUID-PRECIPITATION', 8],
           'AO2': ['LIQUID-PRECIPITATION', 8],
           'AO3': ['LIQUID-PRECIPITATION', 8],
           'AO4': ['LIQUID-PRECIPITATION', 8],
           'AP1': ['15MIN-LIQUID-PRECIP', 6],
           'AP2': ['15MIN-LIQUID-PRECIP', 6],
           'AP3': ['15MIN-LIQUID-PRECIP', 6],
           'AP4': ['15MIN-LIQUID-PRECIP', 6],
           'AT1': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],             # Weather Occurrence Data
           'AT2': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT3': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT4': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT5': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT6': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT7': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AT8': ['DAILY-PRESENT-WEATHER-OBSERVATION', 9],
           'AU1': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU2': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU3': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU4': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU5': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU6': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU7': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU8': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AU9': ['PRESENT-WEATHER-OBSERVATION', 8],
           'AW1': ['PRESENT-WEATHER-OBSERVATION-AUTOMATED', 3],
           'AW2': ['PRESENT-WEATHER-OBSERVATION-AUTOMATED', 3],
           'AW3': ['PRESENT-WEATHER-OBSERVATION-AUTOMATED', 3],
           'AX1': ['PAST-WEATHER-OBSERVATION', 6],
           'AX2': ['PAST-WEATHER-OBSERVATION', 6],
           'AX3': ['PAST-WEATHER-OBSERVATION', 6],
           'AX4': ['PAST-WEATHER-OBSERVATION', 6],
           'AX5': ['PAST-WEATHER-OBSERVATION', 6],
           'AX6': ['PAST-WEATHER-OBSERVATION', 6],
           'AY1': ['PAST-WEATHER-OBSERVATION manual', 5],
           'AY2': ['PAST-WEATHER-OBSERVATION manual', 5],
           'AZ1': ['PAST-WEATHER-OBSERVATION automated', 5],
           'AZ2': ['PAST-WEATHER-OBSERVATION automated', 5],
           'CB1': ['Subhourly Liquid Precipitation Section', 10],       # Climate Reference Network Unique Data
           'CB2': ['Subhourly Liquid Precipitation Section', 10],
           'CF1': ['Hourly Fan Speed Section', 6],
           'CF2': ['Hourly Fan Speed Section', 6],
           'CF3': ['Hourly Fan Speed Section', 6],
           'CG1': ['PRIMARY-SENSOR', 8],
           'CG2': ['PRIMARY-SENSOR', 8],
           'CG3': ['PRIMARY-SENSOR', 8],
           'CH1': ['HOURLY-RELATIVE-HUMIDITY', 15],
           'CH2': ['HOURLY-RELATIVE-HUMIDITY', 15],
           'CI1': ['RELATIVE-HUMIDITY-TEMPERATURE', 28],
           'CN1': ['HOURLY-BATTERY-VOLTAGE', 18],
           'CN2': ['HOURLY-DIAGNOSTIC', 18],
           'CN3': ['SECONDARY-HOURLY-DIAGNOSTIC', 16],
           'CN4': ['SECONDARY-HOURLY-DIAGNOSTIC', 19],
           'CO1': ['US-NETWORK-METADATA', 5],                           # Network Metadata
           'CO2': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO3': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO4': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO5': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO6': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO7': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO8': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CO9': ['US-COOP-NETWORK-TIME-OFFSET', 8],
           'CR1': ['CONTROL-SECTION', 7],
           'CT1': ['SUBHOURLY-TEMPERATURE', 7],
           'CT2': ['SUBHOURLY-TEMPERATURE', 7],
           'CT3': ['SUBHOURLY-TEMPERATURE', 7],
           'CU1': ['HOURLY-TEMPERATURE', 13],
           'CU2': ['HOURLY-TEMPERATURE', 13],
           'CU3': ['HOURLY-TEMPERATURE', 13],
           'CV1': ['HOURLY-TEMPERATURE-EXTREME', 26],
           'CV2': ['HOURLY-TEMPERATURE-EXTREME', 26],
           'CV3': ['HOURLY-TEMPERATURE-EXTREME', 26],
           'CW1': ['SUBHOURLY-WETNESS', 14],
           'CX1': ['HOURLY-GEONOR-VIBRATING-WIRE', 26],
           'CX2': ['HOURLY-GEONOR-VIBRATING-WIRE', 26],
           'CX3': ['HOURLY-GEONOR-VIBRATING-WIRE', 26],
           'ED1': ['RUNWAY-VISUAL-RANGE', 8],                           # Runway Visual Range Data
           'GA1': ['SKY-COVER-LAYER', 13],                              # Cloud and Solar Data
           'GA2': ['SKY-COVER-LAYER', 13],
           'GA3': ['SKY-COVER-LAYER', 13],
           'GA4': ['SKY-COVER-LAYER', 13],
           'GA5': ['SKY-COVER-LAYER', 13],
           'GA6': ['SKY-COVER-LAYER', 13],
           'GD1': ['SKY-COVER-SUMMATION', 12],
           'GD2': ['SKY-COVER-SUMMATION', 12],
           'GD3': ['SKY-COVER-SUMMATION', 12],
           'GD4': ['SKY-COVER-SUMMATION', 12],
           'GD5': ['SKY-COVER-SUMMATION', 12],
           'GD6': ['SKY-COVER-SUMMATION', 12],
           'GE1': ['SKY-CONDITION', 19],
           'GF1': ['SKY-CONDITION', 23],
           'GG1': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG2': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG3': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG4': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG5': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GG6': ['BELOW-STATION-CLOUD-LAYER', 15],
           'GH1': ['HOURLY-SOLAR-RADIATION', 28],
           'GJ1': ['SUNSHINE', 5],
           'GK1': ['SUNSHINE-OBSERVATION', 4],
           'GL1': ['SUSHINE-OBSERVATION-FOR-MONTH', 6],
           'GM1': ['SOLAR-IRRADIANCE', 30],
           'GN1': ['SOLAR-RADIATION', 28],
           'GO1': ['NET-SOLAR-RADIATION', 19],
           'GP1': ['MODELED-SOLAR-IRRADIANCE', 31],
           'GQ1': ['HOURLY-SOLAR-ANGLE', 14],
           'GR1': ['HOURLY-EXTRATERRESTRIAL-RADIATION', 14],
           '???': ['HAIL', 4],                                          # Hail Data - Missing ID in the Doc
           'IA1': ['GROUND-SURFACE', 3],                                # Ground Surface Data
           'IA2': ['GROUND-SURFACE', 9],
           'IB1': ['HOURLY-SURFACE', 27],
           'IB2': ['HOURLY-SURFACE', 13],
           'IC1': ['GROUND-SURFACE', 25],
           'KA1': ['EXTREME-AIR-TEMPERATURE', 10],                      # Temperature Data
           'KA2': ['EXTREME-AIR-TEMPERATURE', 10],
           'KA3': ['EXTREME-AIR-TEMPERATURE', 10],
           'KA4': ['EXTREME-AIR-TEMPERATURE', 10],
           'KB1': ['AVERAGE-AIR-TEMPERATUER', 10],
           'KB2': ['AVERAGE-AIR-TEMPERATUER', 10],
           'KB3': ['AVERAGE-AIR-TEMPERATUER', 10],
           'KC1': ['EXTREME-AIR-TEMP-FOR-MONTH', 14],
           'KC2': ['EXTREME-AIR-TEMP-FOR-MONTH', 14],
           'KD1': ['HEATING-COOLING-DEGREE-DAYS', 9],
           'KD2': ['HEATING-COOLING-DEGREE-DAYS', 9],
           'KE1': ['EXTREME-TEMPS-EXCEEDING-CRITERIA', 12],
           'KF1': ['HOURLY-CALCULATED-TEMP', 6],
           'KG1': ['AVERAGE-DEW_POINT', 11],
           'KG2': ['AVERAGE-DEW_POINT', 11],
           'MA1': ['ATMOSPHERIC-PRESSURE-OBSERVATION', 12],             # Pressure Data
           'MD1': ['ATMOSPHERIC-PRESSURE-CHANGE', 11],
           'ME1': ['GEOPOTENTIAL-HEIGHT', 6],
           'MF1': ['ATMOSPHERIC-PRESSURE-STP-SLP', 12],
           'MG1': ['ATMOSPHERIC-PRESSURE', 12],
           'MH1': ['ATMOSPHERIC-PRESSURE-FOR-MONTH', 12],
           'MK1': ['ATMOSPHERIC-PRESSURE-FOR-MONTH', 24],
           'MV1': ['PRESENT-WEATHER-IN-VICINITY', 3],                   # Weather Occurrence Data (Again?)
           'MV2': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV3': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV4': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV5': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV6': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MV7': ['PRESENT-WEATHER-IN-VICINITY', 3],
           'MW1': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW2': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW3': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW4': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW5': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW6': ['PRESENT-WEATHER-OBSERVATION', 3],
           'MW7': ['PRESENT-WEATHER-OBSERVATION', 3],
           'OA1': ['SUPPLEMENTARY-WIND-OBSERVATION', 8],                # Wind Data
           'OA2': ['SUPPLEMENTARY-WIND-OBSERVATION', 8],
           'OA3': ['SUPPLEMENTARY-WIND-OBSERVATION', 8],
           'OB1': ['Hourly/Sub-Hourly Wind Section', 28],
           'OB2': ['Hourly/Sub-Hourly Wind Section', 28],
           'OC1': ['WIND-GUST-OBSERVATION', 5],
           'OD1': ['SUPPLEMENTARY-WIND-OBSERVATION', 11],
           'OD2': ['SUPPLEMENTARY-WIND-OBSERVATION', 11],
           'OD3': ['SUPPLEMENTARY-WIND-OBSERVATION', 11],
           'RH1': ['RELATIVE HUMIDITY', 9],
           'RH2': ['RELATIVE HUMIDITY', 9],
           'RH3': ['RELATIVE HUMIDITY', 9],
           'SA1': ['SEA-SURFACE-TEMPERATURE', 5],                       # Sea Surface Temperature Data
           'ST1': ['SOIL-TEMPERATURE', 17],                             # Soil Temperature Data
           'UA1': ['WAVE-MEASUREMENT', 10],                             # Marine Data
           'UG1': ['WAVE-MEASUREMENT-PRIMARY', 9],
           'UG2': ['WAVE-MEASUREMENT-SECONDARY', 9],
           'WA1': ['PLATFORM-ICE-ACCRETION', 6],
           'WD1': ['WATER-SURFACE-ICE', 20],
           'WG1': ['WATER-SURFACE-ICE-HISTORICAL', 11],
           'WJ1': ['WATER-LEVEL-OBSERVATION', 19]}


class ISDLineParser:

    def __init__(self, filename, row):

        self.filename = filename
        self.prec_period = None
        self.add_offset = 105  # always fixed
        self.section_chars = 3

        self.varmap = isd_map

        # Include all variables that should appear to every file in __init__ for readability
        # Initialize all to None, then call parsing methods

        self.num_char = None

        # Control section attributes
        self.station_idu = None
        self.station_idn = None
        self.date_ascii = None
        self.time_ascii = None
        self.data_source_flag = None
        self.latitude = None
        self.longitude = None
        self.code = None
        self.elevation = None
        self.call_letter = None
        self.qc_name = None

        # Mandatory section attributes
        # wind-obdervation
        self.direction_angle = None
        self.direction_angle_qc = None
        self.type_code = None
        self.speed = None
        self.speed_qc = None

        # sky-condition
        self.ceiling = None
        self.ceiling_qc = None
        self.ceiling_dc = None

        # visibility
        self.distance = None
        self.distance_qc = None
        self.variability = None
        self.variability_qc = None

        # air-temperature"
        self.air_temp = None
        self.air_temp_qc = None

        # air-temperature dew point"
        self.dpoint_temp = None
        self.dpoint_temp_qc = None

        # atmospheric pressure
        self.sea_level_pressure = None
        self.sea_level_pressure_qc = None

        #non-mandatory field existence checks
        self.have_prec_depth = False
        self.have_cloud = False
        
        # Call the first method to start the initilization of the attributes
        # self.open_file()
        self.parse_file_line(row)

        # Remarks Section
        self.rem_content = None

        # Original Observation Section
        self.has_qnn = False

    def parse_file_line(self, line):
        # First read how many chars the line has
        self.num_char = int(line[0:4]) + self.add_offset
        # Read control and mandatory section
        self.parse_control_section(line)
        self.parse_mandatory_section(line)

        # Additional Data Section
        additional = line[105:108]
        pos = 105
        if additional == "ADD":
            # try to read
            pos += self.section_chars
            while pos < self.num_char:
                try:
                    read = self.parse_add_section(line, pos)
                    pos += read
                except Exception as e:
                    # reached REM
                    # logging.exception(e)
                    break
        while pos < self.num_char:
            if line[pos:pos + self.section_chars] == 'REM':
                try:
                    rem_text = self.parse_rem_section(line[pos:self.num_char])
                    pos += len(rem_text)
                except Exception as e:
                    break
            else:
                break
        if line[pos:pos + self.section_chars] == 'EQD':
            try:
                read = self.parse_eqd_section(line[pos:self.num_char])
                pos += read
            except Exception as e:
                #print(line[pos:pos + self.section_chars])  # change to logging
                pass
        if line[pos:pos + self.section_chars] == 'QNN':
            self.parse_qnn_section()

    def parse_control_section(self, line):
        """Function to read the Control Data Section"""

        # scaling factors
        scale_lats = 1000

        self.station_idu = line[4:10]  # station, USAF catalog identifier
        self.station_idn = line[10:15]  # station, NCEI catalog identifier
        self.date_ascii = line[15:23]  # date of observation
        self.time_ascii = line[23:27]  # time on UTC
        self.datetime_ascii = line[15:27]
        self.data_source_flag = line[27]  
        self.latitude = np.float64(line[28:34]) / scale_lats
        if self.latitude == 99.999:
            self.latitude = np.NaN
        self.longitude = np.float64(line[34:41]) / scale_lats  # 000000 to 179999 are signed negative
        if self.longitude == 999.999:
            self.longitude = np.NaN
        self.code = line[41:46]  # type of geophysical surface observation
        self.elevation = np.float32(line[46:51])  # relative to MSL
        if self.elevation == 9999:
            self.elevation = np.NaN
        # self.call_letter = line[51 - 56]  # no idea
        # self.qc_name = line[56:60]  # quality control process name

    def parse_mandatory_section(self, line):
        """Function to read the Mandatory Data Section"""

        # scaling factors
        scale_speed = 10
        scale_temp = 10
        scale_slp = 10
        
        # constant to convert to kelvin
        kelvin = 273.15

        "Wind-Observation"
        self.direction_angle = float(line[60:63])
        self.direction_angle_qc = line[63]
        if self.direction_angle == 999:
            self.direction_angle_qc = '3'
        self.speed = float(line[65:69]) / scale_speed
        self.speed_qc = line[69]
        if self.speed == 999.9:
            self.speed_qc = '3'

        "Sky-Condition-Observation"
        self.ceiling = float(line[70:75])
        self.ceiling_qc = line[75]
        if self.ceiling == 99999:
            self.ceiling_qc = '3'
        if self.ceiling == 22000:
            self.ceiling_qc = 'CHM'

        "Visibility-Observation"
        self.distance = float(line[78:84])
        self.distance_qc = line[84]
        if self.distance == 999999 or self.distance_qc == 9999:
            self.distance_qc = '3'
        if self.distance == 160000:
            self.distance_qc = 'VIM'

        "Air-Temperature-Observation"
        self.air_temp = (float(line[87:92]) / scale_temp) + kelvin
        self.air_temp_qc = line[92]
        if self.air_temp == 999.9:
            self.air_temp_qc = '3'

        "Air-Temperature-Observation dew point"
        self.dpoint_temp = (float(line[93:98]) / scale_temp) + kelvin
        self.dpoint_temp_qc = line[98]
        if self.dpoint_temp == 999.9:
            self.dpoint_temp_qc = '3'

        "Atmospheric-Pressure-Observation"
        self.sea_level_pressure = float(line[99:104]) / scale_slp
        self.sea_level_pressure_qc = line[104]
        if self.sea_level_pressure == 9999.9:
            self.sea_level_pressure_qc = '3'

    def parse_add_section(self, line, position):
        """ Function to parse the Additional Data Section"""

        identifier = line[position:position + self.section_chars]

        # where the section of data start and end according to the id
        start_block = position + self.section_chars
        end_block = start_block + isd_map[identifier][1]
        block = line[start_block:end_block]

        # the data from the additional section
        # call function find_add_id
        self.find_add_id(identifier, block)

        # send back how many characters were read
        read = isd_map[identifier][1] + self.section_chars
        return read

    def find_add_id(self, identifier, block, print_info=False, print_block=False):
        """ According to the input identifier, the function reads respectively
        the variables under the identifier, following the isd format document.
        Unfortunately, this is by far a not elegant way. Suggestions are welcome."""

        map_value = 0  # position of the name in isd_map
        # scale factor - similar for all values thankfully
        scale_factor = 10
        kelvin = 273.15

        #precipitation accumulation (AO prioritised over AA)
        #both have 4 repeating fields , keep just the first field (will always have the highest frequency of measurement)
        #AO     
        if identifier == 'AO1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.prec_period_ao = float(block[0:2]) # already in minutes
            if self.prec_period_ao == 0:
                return
            self.prec_depth_ao = float(block[2:6]) / scale_factor
            self.prec_depth_ao_qc = block[7]
            if self.prec_depth_ao == 999.9:
                self.prec_depth_ao_qc = '3'

            #check if prec_depth is set, if not set it.
            if not self.have_prec_depth:
                self.prec_depth =    copy.deepcopy(self.prec_depth_ao)
                self.prec_period =   copy.deepcopy(self.prec_period_ao)
                self.prec_depth_qc = copy.deepcopy(self.prec_depth_ao_qc)
                self.have_prec_depth = True
            #else if ao qc code is not suspect/erroneous or data missing, overwrite prec_depth  
            elif (self.prec_depth_ao_qc not in ['2','3','6','7']) & (self.prec_depth_ao != 999.9):
                self.prec_depth =    copy.deepcopy(self.prec_depth_ao)
                self.prec_period =   copy.deepcopy(self.prec_period_ao)
                self.prec_depth_qc = copy.deepcopy(self.prec_depth_ao_qc)
        #AA     
        if identifier == 'AA1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.prec_period_aa = float(block[0:2]) * 60  # because we want minutes
            if self.prec_period_aa == 0:
                return
            self.prec_depth_aa = float(block[2:6]) / scale_factor
            self.prec_depth_aa_qc = block[7]
            if self.prec_depth_aa == 999.9:
                self.prec_depth_aa_qc = '3'

            #check if prec_depth is set, if not set it.
            if not self.have_prec_depth:
                self.prec_depth =    copy.deepcopy(self.prec_depth_aa)
                self.prec_period =   copy.deepcopy(self.prec_period_aa)
                self.prec_depth_qc = copy.deepcopy(self.prec_depth_aa_qc)
                self.have_prec_depth = True
            #else if currently set prec_depth qc code is suspect/erroneous or data is missing, and this is not the case with the aa data, overwrite prec_depth  
            elif (self.prec_depth_qc in ['2','3','6','7']) or (self.prec_depth == 999.9): 
                if (self.prec_depth_aa_qc not in ['2','3','6','7']) & (self.prec_depth_aa != 999.9):
                    self.prec_depth =    copy.deepcopy(self.prec_depth_aa)
                    self.prec_period =   copy.deepcopy(self.prec_period_aa)
                    self.prec_depth_qc = copy.deepcopy(self.prec_depth_aa_qc)

        #snow accumulation
        #4 repeating fields, keep just the first field (will always have the highest frequency of measurement)
        elif identifier == 'AL1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.snow_acc_period = float(block[0:2])  * 60  # because we want minutes
            if self.snow_acc_period == 0:
                return
            self.snow_acc = float(block[2:5])
            self.snow_acc_qc = block[6]
            if self.snow_acc == 999:
                self.snow_acc_qc = '3'

        #snow depth
        elif identifier == 'AJ1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.si = float(block[0:4])
            self.si_qc = block[5]
            if self.si == 9999:
                self.si_qc = '3'

        #relative humidity
        #2 repeating fields, keep just the first field (will always have the highest frequency of measurement)
        elif identifier == 'CH1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.avg_rh_period = float(block[0:2])  # already in minutes
            self.avg_rh = float(block[9:13]) / scale_factor
            self.avg_rh_qc = block[13]
            if self.avg_rh == 999.9:
                self.avg_rh_qc = '3'

        #atmospheric pressure
        elif identifier == 'MA1':
            if print_info:
                print(isd_map[identifier][map_value])
            self.station_p = float(block[6:11]) / scale_factor
            self.station_p_qc = block[11]
            if self.station_p == 9999.9:
                self.station_p_qc = '3'

        #cloud coverage (GA prioritised over GD)
        #GA - coverage of specific cloud layers
        #GD - summary of coverage below specific heights
        #both have 6 repeating fields, keep the field with the maximum valid value

        #GA - coverage provided as oktas
        elif identifier in ('GA1', 'GA2', 'GA3', 'GA4', 'GA5', 'GA6'):
            #first cloud layer
            if identifier == 'GA1':
                self.cloud_ga = int(block[0:2])
                self.cloud_ga_qc = block[2]
                #coverage is not in oktas or missing?
                if self.cloud_ga in [9,10,99]:
                    self.cloud_ga = 99
                    self.cloud_ga_qc = '3'
                
            #have higher coverage value in other layer? 
            elif identifier in ('GA2', 'GA3', 'GA4', 'GA5', 'GA6'):
                new_value = int(block[0:2])
                new_qc = block[2]
                #coverage is not in oktas or missing?
                if new_value in [9,10,99]:
                    new_value = 99
                    new_qc = '3'

                #if current cloud_ga_qc is erroneous/suspect or data is missing, and this is not the case for the new data, then overwrite cloud_ga
                if (self.cloud_ga_qc in ['2','3','6','7']) or (self.cloud_ga == 99): 
                    if (new_qc not in ['2','3','6','7']) & (new_value != 99):
                        self.cloud_ga =    copy.deepcopy(new_value)
                        self.cloud_ga_qc = copy.deepcopy(new_qc)
                #else if both current cloud_ga_qc and new_qc are not erroneous/suspect and data is not missing, then check if new coverage is > cloud_ga
                elif (self.cloud_ga_qc not in ['2','3','6','7']) & (self.cloud_ga != 99) & (new_qc not in ['2','3','6','7']) & (new_value != 99):
                    if new_value > self.cloud_ga:
                        self.cloud_ga = copy.deepcopy(new_value)
                        self.cloud_ga_qc = copy.deepcopy(new_qc)

            #check if cloud is set, if not set it 
            if not self.have_cloud:
                self.cloud =    copy.deepcopy(self.cloud_ga)
                self.cloud_qc = copy.deepcopy(self.cloud_ga_qc)
                if self.cloud == 99:
                    self.cloud_frac = 999
                    self.cloud_frac_qc = '3'
                else:
                    self.cloud_frac = (100.0/8.0)*self.cloud
                    self.cloud_frac_qc = copy.deepcopy(self.cloud_ga_qc)
                self.have_cloud = True
            #else if ga qc code is not suspect/erroneous or data missing, overwrite cloud 
            elif self.cloud_ga_qc not in ['2','3','6','7'] & (self.cloud_ga != 99):
                self.cloud =    copy.deepcopy(self.cloud_ga)
                self.cloud_qc = copy.deepcopy(self.cloud_ga_qc)
                if self.cloud == 99:
                    self.cloud_frac = 999
                    self.cloud_frac_qc = '3'
                else:
                    self.cloud_frac = (100.0/8.0)*self.cloud
                    self.cloud_frac_qc = copy.deepcopy(self.cloud_ga_qc)

        #GD - 2 given coverage codes --> oktas/descriptive and descriptive (which can be converted to oktas)
        elif identifier in ('GD1', 'GD2', 'GD3', 'GD4', 'GD5', 'GD6'):
            #first summation layer
            if identifier == 'GD1':
                #oktas/descriptive coverage code 
                self.cloud_gd = int(block[1:3])
                self.cloud_gd_qc = block[3]
                #convert descriptive codes to oktas
                if self.cloud_gd == 11:
                    self.cloud_gd = 3
                elif self.cloud_gd == 12:
                    self.cloud_gd = 4
                elif self.cloud_gd == 13:
                    self.cloud_gd = 4
                elif self.cloud_gd == 14:
                    self.cloud_gd = 5
                elif self.cloud_gd == 15:
                    self.cloud_gd = 6
                elif self.cloud_gd == 16:
                    self.cloud_gd = 6
                elif self.cloud_gd == 17:
                    self.cloud_gd = 7
                elif self.cloud_gd == 18:
                    self.cloud_gd = 8
                elif self.cloud_gd == 19:
                    self.cloud_gd = 8
                #cannot create coverage code to oktas or missing?
                elif self.cloud_gd in [9,10,99]:
                    #check if descriptive code is valid
                    new_value = int(block[0])
                    #convert code to oktas
                    if new_value == 0:
                        self.cloud_gd = 0
                    elif new_value == 1:
                        self.cloud_gd = 2
                    elif new_value == 2:
                        self.cloud_gd = 4
                    elif new_value == 3:
                        self.cloud_gd = 6
                    elif new_value == 4:
                        self.cloud_gd = 8
                    #cannot create coverage code to oktas or missing?
                    else:
                        self.cloud_gd = 99
                        self.cloud_gd_qc = '3'

            #have higher coverage value in other layer? 
            elif identifier in ('GD2', 'GD3', 'GD4', 'GD5', 'GD6'):
                new_layer_coverage = int(block[1:3])
                new_layer_qc = block[3]
                #convert descriptive codes to oktas
                if new_layer_coverage == 11:
                    new_layer_coverage = 3
                elif new_layer_coverage == 12:
                    new_layer_coverage= 4
                elif new_layer_coverage == 13:
                    new_layer_coverage = 4
                elif new_layer_coverage == 14:
                    new_layer_coverage = 5
                elif new_layer_coverage == 15:
                    new_layer_coverage = 6
                elif new_layer_coverage == 16:
                    new_layer_coverage = 6
                elif new_layer_coverage == 17:
                    new_layer_coverage = 7
                elif new_layer_coverage == 18:
                    new_layer_coverage = 8
                elif new_layer_coverage == 19:
                    new_layer_coverage = 8
                #cannot create coverage code to oktas or missing?
                elif new_layer_coverage in [9,10,99]:
                    #check if descriptive code is valid
                    des_new_layer_coverage = int(block[0])
                    #convert code to oktas
                    if des_new_layer_coverage == 0:
                        new_layer_coverage = 0
                    elif des_new_layer_coverage == 1:
                        new_layer_coverage = 2
                    elif des_new_layer_coverage == 2:
                        new_layer_coverage = 4
                    elif des_new_layer_coverage == 3:
                        new_layer_coverage = 6
                    elif des_new_layer_coverage == 4:
                        new_layer_coverage = 8
                    #cannot create coverage code to oktas or missing?
                    else:
                        new_layer_coverage = 99
                        new_layer_qc = '3'

                #if current cloud_gd_qc is erroneous/suspect or data is missing, and this is not the case for the new data, then overwrite cloud_gd
                if (self.cloud_gd_qc in ['2','3','6','7']) or (self.cloud_gd == 99): 
                    if (new_layer_qc not in ['2','3','6','7']) & (new_layer_coverage != 99):
                        self.cloud_gd =    copy.deepcopy(new_layer_coverage)
                        self.cloud_gd_qc = copy.deepcopy(new_layer_qc)
                #else if both current cloud_gd_qc and new_qc are not erroneous/suspect and data is not missing, then check if new coverage is > cloud_gd
                elif (self.cloud_gd_qc not in ['2','3','6','7']) & (self.cloud_gd != 99) & (new_layer_qc not in ['2','3','6','7']) & (new_layer_coverage != 99):
                    if new_layer_coverage > self.cloud_gd:
                        self.cloud_gd = copy.deepcopy(new_layer_coverage)
                        self.cloud_gd_qc = copy.deepcopy(new_layer_qc)
                
                    
            #check if cloud is set, if not set it 
            if not self.have_cloud:
                self.cloud =    copy.deepcopy(self.cloud_gd)
                self.cloud_qc = copy.deepcopy(self.cloud_gd_qc)
                if self.cloud == 99:
                    self.cloud_frac = 999
                    self.cloud_frac_qc = '3'
                else:
                    self.cloud_frac = (100.0/8.0)*self.cloud
                    self.cloud_frac_qc = copy.deepcopy(self.cloud_gd_qc)
                self.have_cloud = True
            #else if cloud qc code is suspect/erroneous or data missing, and that is not the case with gd data, overwrite cloud 
            elif (self.cloud_qc in ['2','3','6','7']) or (self.cloud == 99):
                if self.cloud_gd_qc not in ['2','3','6','7'] & (self.cloud_gd != 99):
                    self.cloud =    copy.deepcopy(self.cloud_gd)
                    self.cloud_qc = copy.deepcopy(self.cloud_gd_qc)
                    if self.cloud == 99:
                        self.cloud_frac = 999
                        self.cloud_frac_qc = '3'
                    else:
                        self.cloud_frac = (100.0/8.0)*self.cloud
                        self.cloud_frac_qc = copy.deepcopy(self.cloud_gd_qc)

        elif identifier == 'SA1':
            if print_info:
                print(isd_map[identifier][map_value])
            # read part that contains sea surface temperature
            self.sst = (float(block[0:4]) / scale_factor) + kelvin
            self.sst_qc = block[4]
            if self.sst == 99.9:
                self.sst_qc = '3'

        elif identifier == 'ST1':
            if print_info:
                print(isd_map[identifier][map_value])
            # read part that contains soil temperature data
            self.soil_temp = (float(block[1:6]) / scale_factor) + kelvin
            self.soil_temp_qc = block[6]
            if self.soil_temp == 999.9:
                self.soil_temp_depth_qc = '3'
            self.soil_temp_depth = float(block[7:11]) / scale_factor
            self.soil_temp_depth_qc = block[11]
            if self.soil_temp_depth == 999.9:
                self.soil_temp_depth_qc = '3'

       # Read Weather Condition Codes (includes dust-related codes)
        elif identifier in ('AT1', 'AT2', 'AT3', 'AT4', 'AT5', 'AT6', 'AT7', 'AT8'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.at_weather_type{identifier[-1]}_wf = block[2:4]')
            exec(f'self.at_weather_ab_type{identifier[-1]}_wf = block[4:8]')

        elif identifier in ('AU1', 'AU2', 'AU3', 'AU4', 'AU5', 'AU6', 'AU7', 'AU8', 'AU9'):
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.au_proximity{identifier[-1]}_wf = block[0]')
            exec(f'self.au_descriptor{identifier[-1]}_wf = block[1]')
            exec(f'self.au_precip_code{identifier[-1]}_wf = block[2:4]')
            exec(f'self.au_obscuration{identifier[-1]}_wf = block[4]')
            exec(f'self.au_other_weather{identifier[-1]}_wf = block[5]')
            exec(f'self.au_combination{identifier[-1]}_wf = block[6]')
            if print_info:
                print(isd_map[identifier][map_value])

        elif identifier in ('AW1', 'AW2', 'AW3', 'AW4'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.aw_auto_atm_condition{identifier[-1]}_wf = block[0:2]')

        elif identifier in ('MV1', 'MV2', 'MV3', 'MV4', 'MV5', 'MV6', 'MV7'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (6,7):
                return
            exec(f'self.mv_atm_condition{identifier[-1]}_wf = block[0:2]')

        elif identifier in ('MW1', 'MW2', 'MW3', 'MW4', 'MW5', 'MW6', 'MW7'):
            if print_info:
                print(isd_map[identifier][map_value])
            # discard flag if its qa code is defined as invalid
            if block[-1:] in (2,3,6,7):
                return
            exec(f'self.mw_man_atm_condition{identifier[-1]}_wf = block[0:2]')

        if print_block:
            print('read following block: {}'.format(block))

    def parse_rem_section(self, block):
        """Function to read the Remarks Data Section"""

        # initialize position, after REM and fist id
        info = self.section_chars * 2
        # read how many chars the REM has
        to_read = int(block[info:info + self.section_chars]) + self.section_chars * 3  # the ids we read already
        # start position
        pos = 0
        self.rem_content = block[pos:to_read]

        return self.rem_content

    def parse_eqd_section(self, block):
        """Function to read the Element Quality Data Section"""

        param_code = 6  # length of the original value text & parameters' code
        rcode_length = 1  # reason code length
        pos = self.section_chars  # starting from 3, after reading section id
        read = self.section_chars

        while pos < len(block):
            # quality identifier
            qid = block[pos:pos + self.section_chars]
            # if qid == 'QNN':
            #     raise ValueError
            # update position
            pos += self.section_chars
            # original value text
            orig_text = block[pos:pos + param_code]
            # update position
            pos += param_code
            # reason code
            reason_code = block[pos:pos + rcode_length]
            # update position
            pos += rcode_length
            # parameter identifier
            parameter = block[pos:pos + param_code]
            # update position
            pos += param_code

            # characters read so far
            read += len(qid + orig_text + reason_code + parameter)

            if 'WOSPD' in parameter:
                self.speed_orig_value = orig_text
                self.speed_reason_code = reason_code

            if 'WODIR' in parameter:
                self.direction_orig_value = orig_text
                self.direction_reason_code = reason_code

            if 'ATOD' in parameter:
                self.dpoint_orig_value = orig_text
                self.dpoint_reason_code = reason_code

            if 'ATOT' in parameter:
                self.temp_orig_value = orig_text
                self.temp_reason_code = reason_code

            if 'APOSLP' in parameter:
                self.pressure_orig_value = orig_text
                self.pressure_reason_code = reason_code

            if 'SCOCIG' in parameter:
                self.ceiling_orig_value = orig_text
                self.ceiling_reason_code = reason_code

            if 'VODIS' in parameter:
                self.distance_orig_value = orig_text
                self.distance_reason_code = reason_code

        return read

    def parse_qnn_section(self):
        self.has_qnn = True
        # logging.info('You will not believe this! This line has a QNN section! Filename: '.format(self.filename[-17:0]))

class ParseNAPMDFile:

    '''class that handles processing of a NAPMD file'''

    def __init__(self,filename):

        self.filename = filename
        self.start_read = False

        # get time resolution
        self.time_res = self.filename.split('/')[-3]

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row[0] == 'id':
                    column_headers = copy.deepcopy(row)
                    headers_to_alter = {'date_time_utc':'start_dt'}
                    column_headers = [headers_to_alter[column_header] if column_header in list(headers_to_alter.keys()) else column_header for column_header in column_headers]
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                    self.parsed_data['end_dt'] = []
                    self.start_read = True
                #data lines
                elif self.start_read:
                    for column_header, element in zip(column_headers, row):
                        if column_header == 'start_dt':
                            if self.time_res == 'hourly':
                                self.parsed_data['end_dt'].append(datetime.datetime(year=int(element[:4]),month=int(element[5:7]),day=int(element[8:10]),hour=int(element[11:13]),minute=int(element[14:16]))) 
                                self.parsed_data['start_dt'].append(self.parsed_data['end_dt'][-1] - relativedelta(hours=1))
                            elif self.time_res == 'daily':
                                self.parsed_data['end_dt'].append(datetime.datetime(year=int(element[:4]),month=int(element[5:7]),day=int(element[8:10]))) 
                                self.parsed_data['start_dt'].append(self.parsed_data['end_dt'][-1] - relativedelta(days=1))
                        else:
                            self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 


class ParseNZECANFile:

    '''class that handles processing of a NZ ECAN file'''

    def __init__(self,filename,run_details):
        self.filename = filename
        self.run_details = run_details
        self.component_correct = {'CO': 'CO (mg/m3)', 'NO2': 'NO2 (ug/m3)', 'SO2': 'SO2 (ug/m3)', 'NO': 'NO (ug/m3)', 'O3': 'O3 (ug/m3)', 'PM10': 'PM10 (ug/m3)', 'PM2.5': 'PM2.5 (ug/m3)', 'BENZENE': 'Benzene 3M'}

        #get file resolution
        if 'hourly' in self.filename:
            self.file_type = 'hourly'
        elif 'daily' in self.filename:
            self.file_type = 'daily'
        elif '10minute' in self.filename:
            self.file_type = '10minute'

        #self.column_headers = ['start_dt','end_dt', 'CO', 'NO2', 'SO2', 'NO', 'O3', 'PM10', 'PM2.5', 'BENZENE']
        #self.parsed_data = {column_header:[] for column_header in self.column_headers}

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    column_headers_ind = {} # find indices of components in current header of file
                    for column_header in column_headers:
                        if self.run_details['parameter'] in column_header: 
                            column_headers_ind[self.run_details['parameter']] = column_headers.index(self.component_correct[self.run_details['parameter']])

                    
                    column_headers_to_parse = {} # initialize column headers with indices
                    column_headers_to_parse['start_dt'] = -1 
                    
                    if self.run_details['parameter'] in column_headers_ind.keys(): # only parse parameters of run
                        column_headers_to_parse[self.run_details['parameter']] = column_headers_ind[self.run_details['parameter']] #-1,'CO':column_headers.index('CO (mg/m3)'),'NO2':column_headers.index('NO2 (ug/m3)'),'SO2':column_headers.index('SO2 (ug/m3)'),'NO':column_headers.index('NO (ug/m3)'), 'O3':column_headers.index('O3 (ug/m3)'), 'PM10':column_headers.index('PM10 (ug/m3)'), 'PM2.5':column_headers.index('PM2.5 (ug/m3)'), 'BENZENE':column_headers.index('Benzene 3M')}

                    self.parsed_data = {column_header:[] for column_header in column_headers_to_parse}
                    self.parsed_data['end_dt'] = []

                #data lines
                else:
                    for column_header in column_headers_to_parse:
                        if column_header == 'start_dt':
                            if self.file_type == 'daily':
                                self.parsed_data['start_dt'].append(datetime.datetime.strptime(row[0], '%Y-%m-%d'))
                                self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(days=1))
                            elif self.file_type == 'hourly':
                                self.parsed_data['start_dt'].append(datetime.datetime.strptime(row[0], '%d/%m/%Y %I:%M:%S %p'))
                                self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                            elif self.file_type == '10minute':
                                self.parsed_data['start_dt'].append(datetime.datetime.strptime(row[0], '%d/%m/%Y %I:%M:%S %p'))
                                self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(minutes=10))
                            
                        else:
                            self.parsed_data[column_header].append(row[column_headers_to_parse[column_header]]) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)
            
        #return self.parsed_data

class ParseNZLAWAFile:

    '''class that handles processing of a NZ LAWA file'''

    def __init__(self, filename, run_details):

        self.run_details = run_details
        self.filename = filename
        self.variables = ['PM2.5', 'PM10']

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)

                    self.parsed_data = {column_header:[] for column_header in column_headers}
                    self.parsed_data['start_dt'] = []
                    self.parsed_data['end_dt'] = []

                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        if row[9] == self.run_details['parameter']: # only parse rows with parameter
                            self.parsed_data[column_header].append(element)

            # convert times to datetime
            for i in range(len(self.parsed_data['Sample Date'])):
                try:
                    self.parsed_data['start_dt'].append(datetime.datetime.strptime(self.parsed_data['Sample Date'][i], '%Y-%m-%d'))
                except:
                    self.parsed_data['start_dt'].append(datetime.datetime.strptime(self.parsed_data['Sample Date'][i], '%Y-%m-%d %H:%M:%S'))

                self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][i] + relativedelta(days=1))

            # make lists numpy arrays
            self.parsed_data['end_dt'] = np.array(self.parsed_data['end_dt'])
            self.parsed_data['start_dt'] = np.array(self.parsed_data['start_dt'])
                                
            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 

            #return self.parsed_data['start_dt']
        


# class ParseUKAIRFile:
 
#     '''class that handles processing of a UK AIR file, old file format from GHOST 1.5'''
 
#     def __init__(self,filename,uk_air_name):
 
#         self.filename = filename
#         self.parsed_data = {}
#         self.parsed_data['start_dt'] = {}
#         self.parsed_data['end_dt'] = {}
#         self.parsed_data['measurements'] = {}
#         self.parsed_data['units'] = {}
#         self.parsed_data['flags'] = {}
#         self.parsed_data['methods'] = {}
#         self.uk_air_name = uk_air_name.lower()
 
#     def read_data(self):

#         read_active = False
#         station_active = False

#         with open(self.filename, newline='') as csvfile:
#             csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')
#             for row_count, row in enumerate(csvreader):
                
#                 # if have empty row, then continue
#                 if len(row) == 0:
#                     continue

#                 # get station names
#                 elif 'status:' in row[0].lower():
#                     read_active = False
#                     station_active = True
#                 elif 'multi-day data' in row[0].lower():
#                     read_active = False
#                     station_column_names = ['',''] + row[1:]
#                     stations = [col for col in station_column_names if col]
#                 elif station_active:
#                     station_active = False
#                     station_column_names = copy.deepcopy(row)
#                     stations = [col for col in station_column_names if col]
                    
#                 # get column headers
#                 elif (row[0].lower() == 'date') or (row[0].lower() == 'start date'):
#                     column_headers = [col.lower() for col in row]
                    
#                     # reduce stations to only those which have parameter data
#                     # at same time get data indices per station for parameter column
#                     data_indices = {}
#                     station_params = {}
#                     for station_ii, station in enumerate(stations):
#                         # cut column headers to those just relevant for station
#                         station_start_column = station_column_names.index(station)
#                         if station_ii == (len(stations)-1):
#                             station_columns = column_headers[station_start_column:]
#                         else:
#                             station_end_column = station_column_names.index(stations[station_ii+1])
#                             station_columns = column_headers[station_start_column:station_end_column]
#                         # check in current parameter is available for station
#                         if self.uk_air_name in station_columns:
#                             data_indices[station] = station_start_column + station_columns.index(self.uk_air_name)
#                             station_params[station] = self.uk_air_name
#                     stations = list(data_indices.keys())
                    
#                     # if no stations remain, then return from parsing file
#                     if len(stations) == 0:
#                         return

#                     # setup lists for storing data per station
#                     for station in stations: 
#                         for parsed_data_var in self.parsed_data.keys():
#                             if station not in self.parsed_data[parsed_data_var]:
#                                 self.parsed_data[parsed_data_var][station] = []
                    
#                     # get indices for date
#                     start_date_index = None
#                     end_date_index = None
#                     if 'date' in column_headers:
#                         start_date_index = column_headers.index('date')
#                     elif 'start date' in column_headers:
#                         start_date_index = column_headers.index('start date')
#                     if 'end date' in column_headers:
#                         end_date_index = column_headers.index('end date')

#                     # get index for time
#                     time_index = None
#                     if 'time' in column_headers:
#                         time_index = column_headers.index('time')

#                     # get number of time related columns
#                     n_time_columns = 0
#                     dt_indices = [start_date_index, end_date_index, time_index]
#                     for dt_index in dt_indices:
#                         if dt_index is not None:
#                             n_time_columns += 1

#                     # get index for units (per station)
#                     merged_unit_flag = False
#                     unit_indices = {}
#                     if 'unit' in column_headers:
#                         for station in stations:
#                             cut_column_headers = column_headers[data_indices[station]:]
#                             unit_indices[station] = cut_column_headers.index('unit') + data_indices[station]

#                     # get index for flags (per station)
#                     flag_indices = {}
#                     if 'status' in column_headers:
#                         for station in stations:
#                             cut_column_headers = column_headers[data_indices[station]:]
#                             flag_indices[station] = cut_column_headers.index('status') + data_indices[station]
#                         # if have no unit indices then it is because it is merged with flags
#                         if len(unit_indices) == 0:
#                             merged_unit_flag = True
#                             unit_indices = copy.deepcopy(flag_indices)

#                     # get index for units and flags combined (per station)
#                     elif 'status / units' in  column_headers:
#                         merged_unit_flag = True
#                         for station in stations:
#                             cut_column_headers = column_headers[data_indices[station]:]
#                             flag_indices[station] = cut_column_headers.index('status / units') + data_indices[station]
#                         unit_indices = copy.deepcopy(flag_indices)

#                     # set reading to be active
#                     read_active = True

#                 # parse data line (per station)
#                 elif read_active:
                    
#                     # if number of columns does not match the number of column headers then handle it
#                     if len(column_headers) != len(row):

#                         # if have 1 more column header than columns, then this is almost always due to missing units in last column 
#                         # add empty units and continue 
#                         if (len(column_headers) - len(row)) == 1:
#                             row = row + ['']
#                         # if have 1 more column than column headers, then this is almost always due to extra empty data column 
#                         # cut off last column
#                         elif (len(row) - len(column_headers)) == 1:
#                             row = row[:len(column_headers)]
#                         # in some cases the number of data columns is accidentially double and filled with empty data
#                         # in these cases then cut off the empty half of data
#                         elif ((len(column_headers) - n_time_columns) * 2) == len(row): 
#                             row = row[:len(column_headers)]
#                         # otherwise skip line
#                         else:
#                             continue        

#                     # process start/end dt
#                     line_start_date = row[start_date_index]
#                     if '-' in line_start_date:
#                         start_date_split = line_start_date.split('-')
#                     elif '/' in line_start_date:
#                         start_date_split = line_start_date.split('/')
#                     if len(start_date_split[0]) == 4:
#                         start_date_year = int(start_date_split[0]) 
#                         start_date_month = int(start_date_split[1])
#                         start_date_day = int(start_date_split[2])
#                     else:
#                         start_date_year = int(start_date_split[2])
#                         start_date_month = int(start_date_split[1])
#                         start_date_day = int(start_date_split[0])

#                     if end_date_index:
#                         line_end_date = row[end_date_index]
#                         if '-' in line_end_date:
#                             end_date_split = line_end_date.split('-')
#                         elif '/' in line_end_date:
#                             end_date_split = line_end_date.split('/')
#                         if len(end_date_split[0]) == 4:
#                             end_date_year = int(end_date_split[0]) 
#                             end_date_month = int(end_date_split[1])
#                             end_date_day = int(end_date_split[2])
#                         else:
#                             end_date_year = int(end_date_split[2])
#                             end_date_month = int(end_date_split[1])
#                             end_date_day = int(end_date_split[0])

#                     if time_index:
#                         line_time = row[time_index]
#                         time_split = line_time.split(':')
#                         time_hour = int(time_split[0])
#                         time_min = int(time_split[1])

#                     if (start_date_index != None) & (end_date_index != None):
#                         start_dt = datetime.datetime(year=start_date_year, month=start_date_month, day=start_date_day, hour=0, minute=0)
#                         end_dt = datetime.datetime(year=end_date_year, month=end_date_month, day=end_date_day, hour=0, minute=0)
#                     elif (start_date_index != None) & (time_index != None):
#                         # in some cases reported end time will be within 00:00 and 01:00
#                         # in these cases need to remove hour with relativedelta to handle wrapping of days/months/years
#                         time_hour = time_hour - 1
#                         remove_hour = False
#                         if time_hour < 0:
#                             remove_hour = True
#                             time_hour = 0
#                         start_dt = datetime.datetime(year=start_date_year, month=start_date_month, day=start_date_day, hour=time_hour, minute=time_min)
#                         if remove_hour:
#                             start_dt = start_dt - relativedelta(hours=1)
#                         # handle 2 hour vs 1 hour resolution data
#                         if '2H.csv' in self.filename:
#                             delta_hours = 2
#                             start_dt = start_dt - relativedelta(hours=1)
#                         else:
#                             delta_hours = 1

#                         end_dt = start_dt + relativedelta(hours=delta_hours)
#                     elif (start_date_index != None):
#                         start_dt = datetime.datetime(year=start_date_year, month=start_date_month, day=start_date_day, hour=0, minute=0)
#                         end_dt = start_dt + relativedelta(days=1)

#                     # iterate through stations
#                     for station in stations:

#                         # append start/end dt
#                         self.parsed_data['start_dt'][station].append(start_dt)
#                         self.parsed_data['end_dt'][station].append(end_dt)

#                         # parse units
#                         if len(unit_indices) > 0:
#                             station_unit = row[unit_indices[station]] 
#                             if merged_unit_flag:
#                                 if station_unit.strip() == '':
#                                     unit = ''
#                                 else:
#                                     unit = station_unit.split(' ')[1]
#                             else:
#                                 unit = station_unit
#                             if unit.strip().lower() == 'no data':
#                                 unit = ''
#                             self.parsed_data['units'][station].append(unit)
#                         else:
#                             self.parsed_data['units'][station].append('')
                                
#                         # parse flags
#                         if len(flag_indices) > 0:
#                             station_flag = row[flag_indices[station]]
#                             if merged_unit_flag:
#                                 if station_flag.strip() == '':
#                                     flag = ''
#                                 else:
#                                     flag = station_flag.split(' ')[0]
#                             else:
#                                 flag = station_flag
#                             if flag.strip().lower() == 'no data':
#                                 flag = ''
#                             if flag != '':
#                                 self.parsed_data['flags'][station].append(flag)
#                             else:
#                                 self.parsed_data['flags'][station].append('')
#                         else:
#                             self.parsed_data['flags'][station].append('')

#                         # parse data
#                         measurement = row[data_indices[station]]
#                         if measurement.strip().lower() == 'no data':
#                             measurement = ''
#                         #less than (<) or greater than (>) signs are used in file to flag measurements below or above
#                         # detection limits, add flags for this
#                         if '<' in measurement:       
#                             measurement = measurement[1:] 
#                             if self.parsed_data['flags'][station][-1] == '':
#                                 self.parsed_data['flags'][station][-1] = '<'
#                             else:
#                                 self.parsed_data['flags'][station][-1] += '|<'
#                         elif '>' in measurement:
#                             measurement = measurement[1:]  
#                             if self.parsed_data['flags'][station][-1] == '':
#                                 self.parsed_data['flags'][station][-1] = '>'
#                             else:  
#                                 self.parsed_data['flags'][station][-1] += '|>'       
#                         self.parsed_data['measurements'][station].append(measurement)

#                         # parse method (if in units)
#                         if '(' in self.parsed_data['units'][station][-1]:
#                             unit_split = self.parsed_data['units'][station][-1].split(' ')
#                             self.parsed_data['units'][station][-1] = unit_split[0]
#                             self.parsed_data['methods'][station].append(' '.join(unit_split[1:])[1:-1])
#                         # handle some special cases
#                         elif 'NH3_active.csv' in self.filename:
#                             self.parsed_data['methods'][station].append('nh3active')
#                         elif 'NH3_diffusiontube.csv' in self.filename:
#                             self.parsed_data['methods'][station].append('nh3diffusiontube')
#                         elif 'NH3_passive.csv' in self.filename:
#                             self.parsed_data['methods'][station].append('nh3passive')
#                         elif ('reflectance' in station_params[station]) or ('-TOR-' in station_params[station]):
#                             self.parsed_data['methods'][station].append('improve-a')
#                         elif ('transmittance' in station_params[station]) or ('-TOT-' in station_params[station]):
#                             self.parsed_data['methods'][station].append('niosh-5040')
#                         else:
#                             self.parsed_data['methods'][station].append('')

#             #convert lists to numpy arrays
#             for column_header in list(self.parsed_data.keys()):
#                 if column_header in ['start_dt','end_dt']:
#                     for station in self.parsed_data[column_header].keys():
#                         self.parsed_data[column_header][station] = np.array(self.parsed_data[column_header][station])
#                 else:
#                     for station in self.parsed_data[column_header].keys():
#                         self.parsed_data[column_header][station] = np.array(self.parsed_data[column_header][station], dtype=np.str_)             

class ParseUKAIRFile:
 
    '''class that handles processing of a UK AIR file, new file format .xml from GHOST 1.5.1'''
 
    def __init__(self,filename,run_details, parameter_details):
 
        self.filename = filename
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.parsed_data = {}
        self.parsed_metadata = {}
 
    def read_data(self):
        
        # parse xml file
        tree = ET.parse(self.filename)
        root = tree.getroot()

        # list of parameters in single file: list all the parameters in order to avoid duplicates
        parameters_per_file = []
        
        # extract namespaces to find elements
        namespaces = dict([node for _, node in ET.iterparse(self.filename, events=['start-ns'])])

        for member in root.findall('gml:featureMember', namespaces):
            # find observations in feature members, usually none or one per member
            for observation in member.findall('om:OM_Observation', namespaces):
                #gml_id = observation.attrib['{http://www.opengis.net/gml/3.2}id'].split('_')[-1].strip()

                # find measured parameter and select matching data in file, otherwise skip
                if 'eionet' in list(observation.find('om:observedProperty', namespaces).attrib.values())[0]: # check if information is given as eionet link what component it is
                    parameter_code = observation.find('om:observedProperty', namespaces).attrib['{http://www.w3.org/1999/xlink}href'].split('/')[-1].zfill(5)

                    if parameter_code == self.parameter_details['airbase_code']:
                        self.parsed_metadata['parameter_code'] = parameter_code

                        # check if parameter has already been parsed from the same file
                        if parameter_code in parameters_per_file:
                            continue
                        # add parameter to list as parsed
                        parameters_per_file.append(parameter_code)

                        # find some metadata parameters
                        for parameter in observation.findall('om:parameter', namespaces):
                            name = parameter.find('om:NamedValue', namespaces).find('om:name', namespaces).attrib['{http://www.w3.org/1999/xlink}href']
                            value = parameter.find('om:NamedValue', namespaces).find('om:value', namespaces).attrib['{http://www.w3.org/1999/xlink}href']
                            self.parsed_metadata[name.split('/')[-1]] = value.split('/')[-1].split('_')[-1]

                        # if gml_id and sampling point match, take data
                        #if gml_id == self.parsed_metadata['SamplingPoint']:
                        # find data
                        measurement = observation.find('om:result', namespaces).find('swe:DataArray', namespaces)

                        for field in measurement.find('swe:elementType', namespaces).find('swe:DataRecord', namespaces).findall('swe:field', namespaces):
                            self.parsed_data[field.attrib['name']] = []

                            # find unit
                            if field.attrib['name'] == 'Value':
                                raw_unit = field.find('swe:Quantity', namespaces).find('swe:uom', namespaces).attrib['{http://www.w3.org/1999/xlink}href'].split('/')[-1]
                                
                                if '_PREC' in self.run_details['parameter']:
                                    if self.run_details['parameter'] == 'H+_PREC':
                                        self.parsed_metadata['unit'] = 'ueq l-1'  # assume Eionet standard units since unit is not given in file, huge assumption!
                                    else: 
                                        self.parsed_metadata['unit'] = 'mg l-1' # assume Eionet standard units since unit is not given in file, huge assumption!
                                elif self.run_details['parameter'] == 'PREC_ACCUM':
                                    self.parsed_metadata['unit'] = 'mm' # assume Eionet standard units since unit is not given in file, huge assumption!
                                else:
                                    self.parsed_metadata['unit'] = raw_unit

                        # find separators
                        dec_separator = measurement.find('swe:encoding', namespaces).find('swe:TextEncoding', namespaces).attrib['decimalSeparator']
                        tok_separator = measurement.find('swe:encoding', namespaces).find('swe:TextEncoding', namespaces).attrib['tokenSeparator']
                        blo_separator = measurement.find('swe:encoding', namespaces).find('swe:TextEncoding', namespaces).attrib['blockSeparator']

                        # parse measurements
                        values = str(measurement.find('swe:values', namespaces).text)
                        single_measurements = values.split(str(blo_separator))
                        
                        for single_measurement in single_measurements:
                            if single_measurement != '': # skip empty rows
                                columns = single_measurement.split(str(tok_separator))

                                for ii, header in enumerate(self.parsed_data):
                                    if header in ['StartTime', 'EndTime']:
                                        raw_date = columns[ii].strip()
                                        if '24:00:00' in raw_date: # replace 24:00:00 by 00:00:00 and add one day
                                            raw_date = raw_date.replace('24:00:00', '00:00:00')
                                            date_time = datetime.datetime.strptime(str(raw_date), '%Y-%m-%dT%H:%M:%SZ') + relativedelta(days=1)
                                        else:
                                            date_time = datetime.datetime.strptime(str(raw_date), '%Y-%m-%dT%H:%M:%SZ')

                                        self.parsed_data[header].append(date_time)
                                    else:
                                        self.parsed_data[header].append(columns[ii].strip())
                                
                        # rename start and end time columns
                        self.parsed_data['start_dt'] = self.parsed_data.pop('StartTime')
                        self.parsed_data['end_dt'] = self.parsed_data.pop('EndTime')

                        #convert lists to numpy arrays
                        for column_header in list(self.parsed_data.keys()):
                            if column_header in ['start_dt','end_dt']:
                                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                            else:
                                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)

        return self.parsed_metadata



class ParseUSEPAAirNowDOS:

    '''class that handles processing of a US EPA AirNow DOS file'''

    def __init__(self,filename,run_details):
        self.filename = filename
        self.run_details = run_details

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    column_headers_to_parse = {'start_dt':-1,'measurement':column_headers.index('Raw Conc.'),'unit':column_headers.index('Conc. Unit'),'qa':column_headers.index('QC Name')}
                    column_headers_to_process = {'year':column_headers.index('Year'), 'month':column_headers.index('Month'), 'day':column_headers.index('Day'), 'hour':column_headers.index('Hour'), 'duration':column_headers.index('Duration')}
                    self.parsed_data = {column_header:[] for column_header in column_headers_to_parse}
                    self.parsed_data['end_dt'] = []
                #data lines
                else:
                    for column_header in column_headers_to_parse:
                        if column_header == 'start_dt':
                            year = int(row[column_headers_to_process['year']])
                            month = int(row[column_headers_to_process['month']])
                            day = int(row[column_headers_to_process['day']])
                            hour = int(row[column_headers_to_process['hour']])
                            duration = row[column_headers_to_process['duration']]

                            if duration != '1 Hr':
                                self.run_details['log_file_str'] += 'NOT HOURLY RESOLUTION IN FILE: {} .STOPPING PROCESSING.'.format(self.filename)
                                create_GHOST_stage_logfile(2,self.run_details)

                            self.parsed_data['start_dt'].append(datetime.datetime(year=year,month=month,day=day,hour=hour,minute=0)) 
                            self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][-1] + relativedelta(hours=1))
                        else:
                            self.parsed_data[column_header].append(row[column_headers_to_parse[column_header]]) 

            #convert lists to numpy arrays
            for column_header in list(self.parsed_data.keys()):
                if column_header in ['start_dt','end_dt']:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
                else:
                    self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 


class ParseUSEPAAQSFile:

    '''class that handles processing of a US EPA AQS file'''

    def __init__(self,filename, run_details):

        self.filename = filename
        self.run_details = run_details
        #get resolution of file
        self.file_resolution = self.filename.split('/')[-1].split('_')[0]
        self.parsed_data = {'Parameter_name': [], 'start_dt': [], 'end_dt':[], 'Measurement': [], 'Units': [], 'Sample_duration': [], 'Detection_limit': [], 'Method_type': [], 'Method_code':[], 'Qualifier':[], 'Uncertainty': [], 'Longitude': [], 'Latitude': []}

    def read_data(self):

        with open(self.filename, 'r', encoding='utf-8') as f:
            file_data = json.loads(f.read())

        raw_data = file_data['Data']
        for i, datapoint in enumerate(raw_data):
            self.parsed_data['Parameter_name'].append(datapoint['parameter'])
            self.parsed_data['Units'].append(datapoint['units_of_measure'])
            self.parsed_data['Longitude'].append(datapoint['longitude'])
            self.parsed_data['Latitude'].append(datapoint['latitude'])
            self.parsed_data['Sample_duration'].append(datapoint['sample_duration'])
            self.parsed_data['Detection_limit'].append(datapoint['detection_limit'])
            self.parsed_data['Uncertainty'].append(datapoint['uncertainty'])
            self.parsed_data['Method_type'].append(datapoint['method_type'])
            self.parsed_data['Method_code'].append(datapoint['method_code'])
            self.parsed_data['Qualifier'].append(str(datapoint['qualifier']).split('-')[0].strip())
            if self.run_details['parameter'] == 'H+_PREC':
                if datapoint['sample_measurement'] != None:
                    self.parsed_data['Measurement'].append(str(10**-float(datapoint['sample_measurement']))) # convert pH to H+ concentration in mol l-1
                else:
                    self.parsed_data['Measurement'].append(datapoint['sample_measurement'])
            else:
                self.parsed_data['Measurement'].append(datapoint['sample_measurement'])

            # date time
            year = int(datapoint['date_local'].split('-')[0])
            month = int(datapoint['date_local'].split('-')[1])
            day = int(datapoint['date_local'].split('-')[2])
            hour = int(datapoint['time_local'].split(':')[0])
            minute = int(datapoint['time_local'].split(':')[1])

            self.parsed_data['start_dt'].append(datetime.datetime(year=year,month=month,day=day,hour=hour,minute=minute))
            if self.parsed_data['Sample_duration'][i] == '1 WEEK':
                self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][i] + relativedelta(days=7))
            elif self.parsed_data['Sample_duration'][i] == '1 HOUR':
                self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][i] + relativedelta(hours=1))
            elif self.parsed_data['Sample_duration'][i] == '24 HOUR':
                self.parsed_data['end_dt'].append(self.parsed_data['start_dt'][i] + relativedelta(days=1))

        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)

        #return self.parsed_data      

# class ParseUSEPAAQSFile:

#     '''class that handles processing of a US EPA AQS file'''

#     def __init__(self,filename,parameter_codes):

#         self.filename = filename
#         self.parameter_codes = parameter_codes
#         #get resolution of file
#         self.file_resolution = self.filename.split('/')[-1].split('_')[0]
#         #get number of lines in file (for memory allocation)
#         with open(self.filename) as f:
#             n_lines = sum(1 for line in f) - 1
#         #set column headers per file resolution
#         if self.file_resolution == 'hourly':
#             self.column_headers = ['station_reference','start_datetime','measurement','Parameter Code','POC','Latitude','Longitude','Datum','Units of Measure','MDL','Uncertainty','Qualifier','Method Code']
#         elif self.file_resolution == 'daily':
#             self.column_headers = ['station_reference','start_datetime','measurement','Parameter Code','POC','Latitude','Longitude','Datum','Units of Measure','Observation Percent','Method Code']
#         #set empty dataframe to store parsed data
#         standard_dtype = [(key,object) for key in self.column_headers]
#         self.parsed_data = np.empty(n_lines, dtype=standard_dtype)

#     def read_data(self):

#         parsed_lines = 0
#         with open(self.filename, newline='') as csvfile:
#             csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

#             for row_count, row in enumerate(csvreader):
                
#                 #get indices associated with fields that need manipulating from first line
#                 if row_count == 0:
#                    state_code_index = row.index('State Code') 
#                    county_code_index = row.index('County Code') 
#                    site_num_index = row.index('Site Num')
#                    if self.file_resolution == 'hourly':
#                        date_index = row.index('Date GMT')
#                        time_index = row.index('Time GMT')
#                        measurement_index = row.index('Sample Measurement')
#                    elif self.file_resolution == 'daily':
#                        date_index = row.index('Date Local')
#                        measurement_index = row.index('Arithmetic Mean')
#                    original_column_headers = copy.deepcopy(row)
#                    #get indices for fields which do not need to manipulate
#                    column_indices = {}
#                    for header_ii, original_column_header in enumerate(original_column_headers):
#                        if original_column_header in self.column_headers:
#                            column_indices[original_column_header] = header_ii
                   
#                 #otherwise parse line    
#                 else:
#                     #only parse line if parameter code is for current species
#                     if row[column_indices['Parameter Code']] in self.parameter_codes:
                        
#                         #generate station reference (State Code + County Code + Site Num)
#                         self.parsed_data['station_reference'][parsed_lines] = row[state_code_index] + row[county_code_index] + row[site_num_index]

#                         #parse datetime
#                         split_date = row[date_index].split('-')
#                         if self.file_resolution == 'hourly':
#                             split_time = row[time_index].split(':')
#                             self.parsed_data['start_datetime'][parsed_lines] = datetime.datetime(year=int(split_date[0]), month=int(split_date[1]), day=int(split_date[2]), hour=int(split_time[0]), minute=int(split_time[1]))
#                         elif self.file_resolution == 'daily':
#                             self.parsed_data['start_datetime'][parsed_lines] = datetime.datetime(year=int(split_date[0]), month=int(split_date[1]), day=int(split_date[2]), hour=0, minute=0)                       

#                         #parse measurements
#                         self.parsed_data['measurement'][parsed_lines] = row[measurement_index]
                        
#                         #parse rest of fields
#                         for column_header in column_indices.keys():
#                             self.parsed_data[column_header][parsed_lines] = row[column_indices[column_header]]

#                         #add to parsed line counter
#                         parsed_lines += 1

#         #delete empty lines
#         if parsed_lines < len(self.parsed_data):
#             self.parsed_data = self.parsed_data[:parsed_lines]
                

class ParseUSEPACASTNETFile:

    '''class that handles processing of a US EPA CASTNET file'''

    def __init__(self,filename):

        self.filename = filename

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        if column_header == 'PH': # pH as H+
                            if element != '':
                                self.parsed_data[column_header].append(str(10**(-float(element)))) # convert pH to H+ concentration in mol l-1
                            else:
                                self.parsed_data[column_header].append('')
                        else:
                            self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)
           

class ParseUSNADPAMNetFile:

    '''class that handles processing of a US NADP AMNet file'''

    def __init__(self,filename):

        self.filename = filename

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 

            #convert missing measurements to NaN 
            self.parsed_data['PBM'] = self.parsed_data['PBM'].astype(np.float32)
            missing_inds = np.where(self.parsed_data['PBM'] == -9.0)[0]
            self.parsed_data['PBM'][missing_inds] = np.NaN 

            
class ParseUSNADPAMoNFile:

    '''class that handles processing of a US NADP AMoN file'''

    def __init__(self,filename):

        self.filename = filename

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    self.parsed_data = {column_header:[] for column_header in column_headers}
                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        self.parsed_data[column_header].append(element) 

            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 

            #convert missing measurements to NaN 
            self.parsed_data['CONC'] = self.parsed_data['CONC'].astype(np.float32)
            missing_inds = np.where(self.parsed_data['CONC'] == -9.0)[0]
            self.parsed_data['CONC'][missing_inds] = np.NaN 


class ParseUSNADPFile:

    '''class that handles processing of a US NADP file'''

    def __init__(self,filename, run_details):

        self.filename = filename
        self.run_details = run_details
        self.variables = {'AMNET': ['PBM', 'GOM', 'GEM'], 
                          'AMoN': ['CONC'], 
                          'NTN': ['Ca++_PREC', 'Mg++_PREC', 'K+_PREC', 'Na+_PREC', 'NH4+_PREC', 'NO3-_PREC', 'Cl-_PREC', 'SO4--_PREC','H+_PREC','PREC_ACCUM'],
                          'MDN': ['Hg_PREC', 'PREC_ACCUM'],
                          'AIRMoN': ['Ca++_PREC', 'Mg++_PREC', 'K+_PREC', 'Na+_PREC', 'NH4+_PREC', 'NO3-_PREC', 'Cl-_PREC', 'SO4--_PREC', 'PO4---_PREC', 'H+_PREC','PREC_ACCUM']}

    def read_data(self):

        # find correct variable for NADP network
        for var in self.variables:
            if var in self.filename:
                variable = self.variables[var]

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

            for row_count, row in enumerate(csvreader):

                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    
                    if 'NTN' in self.filename: # correct header to GHOST standards for NTN network
                        #correct column header names to GHOST standards
                        to_correct = {'Ca':'Ca++_PREC','Mg':'Mg++_PREC', 'K': 'K+_PREC', 'na':'Na+_PREC', 'NH4':'NH4+_PREC', 'NO3': 'NO3-_PREC', 'Cl': 'Cl-_PREC', 'SO4':'SO4--_PREC', 'flagCa':'Ca++_PREC_DL','flagMg':'Mg++_PREC_DL', 'flagK': 'K+_PREC_DL', 'flagNa':'Na+_PREC_DL', 'flagNH4':'NH4+_PREC_DL', 'flagNO3': 'NO3-_PREC_DL', 'flagCl': 'Cl-_PREC_DL', 'flagSO4':'SO4--_PREC_DL', 'ph':'H+_PREC','subppt': 'PREC_ACCUM'}
                        for to_c in to_correct.keys():
                            if to_c in column_headers:
                                column_headers[column_headers.index(to_c)] = to_correct[to_c]
                    
                    if 'MDN' in self.filename: # correct header to GHOST standards for MDN network
                        #correct column header names to GHOST standards
                        to_correct = {'HgConc':'Hg_PREC','subppt': 'PREC_ACCUM'}
                        for to_c in to_correct.keys():
                            if to_c in column_headers:
                                column_headers[column_headers.index(to_c)] = to_correct[to_c]

                    if 'AIRMoN' in self.filename: # correct header to GHOST standards for AIRMoN network
                        #correct column header names to GHOST standards
                        to_correct = {'Ca':'Ca++_PREC','Mg':'Mg++_PREC', 'K': 'K+_PREC', 'Na':'Na+_PREC', 'NH4':'NH4+_PREC', 'NO3': 'NO3-_PREC', 'Cl': 'Cl-_PREC', 'SO4':'SO4--_PREC', 'PO4': 'PO4---_PREC','pHlab': 'H+_PREC', 'subppt': 'PREC_ACCUM'}
                        for to_c in to_correct.keys():
                            if to_c in column_headers:
                                column_headers[column_headers.index(to_c)] = to_correct[to_c]

                    self.parsed_data = {column_header:[] for column_header in column_headers}

                #data lines
                else:
                    for column_header, element in zip(column_headers, row):
                        if (self.run_details['parameter'] in ['H+_PREC']) and (column_header == 'H+_PREC'):
                            if (float(element) > 0) and (element != ''):
                                self.parsed_data[column_header].append(10**-float(element)) # convert pH to H+ concentration in mol/l
                            else:
                                self.parsed_data[column_header].append(element) # parse -9.0 flags as normal
                        else:
                            self.parsed_data[column_header].append(element) 


            #convert lists to numpy arrays
            for column_header in column_headers:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 

            #convert missing measurements to NaN 
            for var in variable:
                if var in ['PREC_ACCUM']:
                    if 'NTN' in self.filename:
                        self.parsed_data[var] = self.parsed_data[var].astype(np.float32)
                        missing_inds = np.where(self.parsed_data[var] == -9.99)[0]
                        self.parsed_data[var][missing_inds] = np.NaN
                    if 'MDN' in self.filename:
                        self.parsed_data[var] = self.parsed_data[var].astype(np.float32)
                        missing_inds = np.where(self.parsed_data[var] == -9.0)[0]
                        self.parsed_data[var][missing_inds] = np.NaN
                else:
                    self.parsed_data[var] = self.parsed_data[var].astype(np.float32)
                    missing_inds = np.where(self.parsed_data[var] == -9.0)[0]
                    self.parsed_data[var][missing_inds] = np.NaN
            
            # parse flags in AIRMoN to lower case, and ensure qrcode to be upper case
            if 'AIRMoN' in self.filename:
                for ii in range(len(self.parsed_data['sl'])):
                    self.parsed_data['sl'][ii] = self.parsed_data['sl'][ii].lower()
                    self.parsed_data['qrcode'][ii] = self.parsed_data['qrcode'][ii].upper()



class ParseWMOWDCGGFile:

    '''class that handles processing of a WMO WDCGG file'''

    def __init__(self,filename):

        self.filename = filename
        self.read_header_global = False
        self.read_header_variable = False
        self.read_header_global_start = -999
        self.read_header_variable_start = -999

        self.global_attributes = {}
        self.variable_attributes = {} 
        self.parsed_data = {}
         

    def read_metadata(self):

        #read header
        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=':', quoting=csv.QUOTE_NONE)
            for row_count, row in enumerate(csvreader):

                if row_count == 0:
                    self.n_header_lines = int(row[-1].strip())
                elif row_count == self.read_header_global_start:
                    self.read_header_global = True
                    self.read_header_variable = False
                elif row_count == self.read_header_variable_start:
                    self.read_header_global = False
                    self.read_header_variable = True
                elif row_count == (self.n_header_lines-1):
                    self.read_header_global = False
                    self.read_header_variable = False
                    break

                if '# GLOBAL ATTRIBUTES' in row:
                    self.read_header_global_start = row_count + 2
        
                elif '# VARIABLE ATTRIBUTES' in row:
                    self.read_header_variable_start = row_count + 2
               
                #parse global header attributes
                if self.read_header_global:
                    if len(row) >= 2:
                        if row[1].strip() == 'comment':
                            global_attribute_name = '{}:{}'.format(row[0][1:].strip(), row[1].strip())
                            if len(row) > 3:
                                global_attribute_value = ':'.join([row_value.strip() for row_value in row[2:]])
                            else:
                                global_attribute_value = row[2].strip()
                        else:
                            global_attribute_name = row[0][1:].strip()
                            if len(row) > 2:
                                global_attribute_value = ':'.join([row_value.strip() for row_value in row[1:]])
                            else:
                                global_attribute_value = row[1].strip()
                        #assign value
                        if global_attribute_name in list(self.global_attributes.keys()):
                            self.global_attributes[global_attribute_name] = '{};{}'.format(self.global_attributes[global_attribute_name], global_attribute_value)
                        else:
                            self.global_attributes[global_attribute_name] = copy.deepcopy(global_attribute_value)

                #parse variable header attributes
                elif self.read_header_variable:
                    if len(row) >= 3:
                        variable_attribute_name = '{}:{}'.format(row[0][1:].strip(), row[1].strip())
                        if len(row) > 3:
                            variable_attribute_value = ':'.join([row_value.strip() for row_value in row[2:]])
                        else:
                            variable_attribute_value = row[2].strip()
                        #assign value
                        if variable_attribute_name in list(self.variable_attributes.keys()):
                            self.variable_attributes[variable_attribute_name] = '{};{}'.format(self.variable_attributes[variable_attribute_name], variable_attribute_value)
                        else:
                            self.variable_attributes[variable_attribute_name] = copy.deepcopy(variable_attribute_value)

    def read_data(self):

        #read data
        with open(self.filename, newline='', encoding='utf-8') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=' ', quoting=csv.QUOTE_NONE)
            for row_count, row in enumerate(csvreader):
            
                if row_count == (self.n_header_lines-1):
                    column_headers_orig = ['site_gaw_id','start_year','start_month','start_day','start_hour','start_minute','start_second','end_year','end_month','end_day','end_hour','end_minute','end_second','value','value_unc','nvalue','latitude','longitude','altitude','elevation','intake_height','flask_no','ORG_QCflag','QCflag','instrument','measurement_method','scale']
                    column_headers_new = ['start_dt','end_dt','value','value_unc','latitude','longitude','elevation','intake_height','QCflag','instrument','measurement_method','scale']
                    for column_header in column_headers_new:
                        self.parsed_data[column_header] = []

                    #get indices for dates
                    start_year_index = column_headers_orig.index('start_year')
                    start_month_index = column_headers_orig.index('start_month')
                    start_day_index = column_headers_orig.index('start_day')      
                    start_hour_index = column_headers_orig.index('start_hour')
                    start_minute_index = column_headers_orig.index('start_minute')
                    start_second_index = column_headers_orig.index('start_second')

                    end_year_index = column_headers_orig.index('end_year')
                    end_month_index = column_headers_orig.index('end_month')
                    end_day_index = column_headers_orig.index('end_day')
                    end_hour_index = column_headers_orig.index('end_hour')
                    end_minute_index = column_headers_orig.index('end_minute')
                    end_second_index = column_headers_orig.index('end_second')

                elif row_count >= self.n_header_lines:
                    #parse start_dt
                    start_year = int(row[start_year_index])
                    start_month = int(row[start_month_index])
                    start_day = int(row[start_day_index])
                    start_hour = int(row[start_hour_index])
                    start_minute = int(row[start_minute_index])
                    start_second = int(row[start_second_index])
                    if (start_year >= 0) & (start_month >= 0) & (start_day >= 0) & (start_hour >= 0) & (start_minute >= 0) & (start_second >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=start_hour,minute=start_minute,second=start_second)
                    elif (start_year >= 0) & (start_month >= 0) & (start_day >= 0) & (start_hour >= 0) & (start_minute >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=start_hour,minute=start_minute)
                    elif (start_year >= 0) & (start_month >= 0) & (start_day >= 0) & (start_hour >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=start_hour,minute=0)
                    elif ('_daily' in self.filename) & (start_year >= 0) & (start_month >= 0) & (start_day >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=start_day,hour=0,minute=0)
                    elif ('_monthly' in self.filename) & (start_year >= 0) & (start_month >= 0):
                        s_dt = datetime.datetime(year=start_year,month=start_month,day=1,hour=0,minute=0)
                    #continue to next row as cannot parse start time
                    else:
                        continue                         

                    #parse end dt (if not given, make from stated sampling frequency/file resolution - if event, assume sampling is 1 minute)
                    end_year = int(row[end_year_index])
                    end_month = int(row[end_month_index])
                    end_day = int(row[end_day_index])
                    end_hour = int(row[end_hour_index])
                    end_minute = int(row[end_minute_index])
                    end_second = int(row[end_second_index])
                    if (end_year >= 0) & (end_month >= 0) & (end_day >= 0) & (end_hour >= 0) & (end_minute >= 0) & (end_second >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=end_hour,minute=end_minute,second=end_second)
                    elif (end_year >= 0) & (end_month >= 0) & (end_day >= 0) & (end_hour >= 0) & (end_minute >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=end_hour,minute=end_minute)
                    elif (end_year >= 0) & (end_month >= 0) & (end_day >= 0) & (end_hour >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=end_hour,minute=0)
                    elif ('_daily' in self.filename) & (end_year >= 0) & (end_month >= 0) & (end_day >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=end_day,hour=0,minute=0)
                    elif ('_monthly' in self.filename) & (end_year >= 0) & (end_month >= 0):
                        e_dt = datetime.datetime(year=end_year,month=end_month,day=1,hour=0,minute=0)
                    #if not end time provided, generate from file resolution
                    else:
                        if '_hourly' in self.filename:
                            e_dt = s_dt + relativedelta(hours=1)
                        elif '_daily' in self.filename:
                            e_dt = s_dt + relativedelta(days=1)
                        elif '_monthly' in self.filename:
                            e_dt = s_dt + relativedelta(months=1)
                        elif '_event' in self.filename:
                            e_dt = s_dt + relativedelta(minutes=1)
                            
                    #append start_dt/end_dt         
                    self.parsed_data['start_dt'].append(s_dt)
                    self.parsed_data['end_dt'].append(e_dt)

                    #now append rest of row 
                    for column_header, row_value in zip(column_headers_orig, row):
                        if column_header not in ['site_gaw_id','nvalue','altitude','flask_no','ORG_QCflag','start_year','start_month','start_day','start_hour','start_minute','start_second','end_year','end_month','end_day','end_hour','end_minute','end_second']:
                            self.parsed_data[column_header].append(row_value) 


        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_) 
                
                #set fill values as empty strings
                nan_inds = np.where(self.parsed_data[column_header] == self.variable_attributes['{}:_FillValue'.format(column_header)])[0]                        
                self.parsed_data[column_header][nan_inds] = ''


class ParseWMOWDCPCFile:

    '''class that handles processing of a WMO WDCPC file'''

    def __init__(self,filename,run_details):
        self.filename = filename
        self.run_details = run_details
        self.component_correct = {'Ca++_PREC': 'Ca', 'Mg++_PREC': 'Mg', 'K+_PREC': 'K', 'Na+_PREC': 'Na', 'NH4+_PREC': 'NH4', 'NO3-_PREC': 'NO3', 'Cl-_PREC': 'Cl', 'SO4--_PREC': 'SO4', 'PO4---_PREC': 'PO4', 'F-_PREC': 'F', 'Zn_PREC': 'Zn', 'H+_PREC': 'Acidity', 'PREC_ACCUM': 'ACCUM'}
        self.parsed_data = {}

    def read_data(self):

        with open(self.filename, newline='') as csvfile:
            csvreader = csv.reader(csvfile, delimiter=',')
            
            csvfile.seek(0)
            for row_count, row in enumerate(csvreader):
                #column headers
                if row_count == 0:
                    column_headers = copy.deepcopy(row)
                    column_headers_ind = {} # find indices of components in current header of file
                    for column_header in column_headers:
                        # correct Guage_Depth column to fit to other column names
                        if column_header == 'Guage_Depth':
                            index = column_headers.index(column_header)
                            column_headers[index] = 'ACCUM'
                        if column_header == 'Guage_Flag':
                            index = column_headers.index(column_header)
                            column_headers[index] = 'ACCUM_Flag'
                    
                    for column_header in column_headers:
                        # find parameter and flags of the run 
                        if self.run_details['parameter'].replace('+', '').replace('-', '').replace('PREC', '').replace('_', '') in column_header:
                            if '_Flag' in column_header: # flags
                                if '_DL_Flag' in column_header: # dl flags
                                    column_headers_ind[self.run_details['parameter']+'_dl_flag'] = column_headers.index(self.component_correct[self.run_details['parameter']]+'_DL_Flag')
                                else:
                                    column_headers_ind[self.run_details['parameter']+'_flag'] = column_headers.index(self.component_correct[self.run_details['parameter']]+'_Flag')
                            else: # data
                                column_headers_ind[self.run_details['parameter']] = column_headers.index(self.component_correct[self.run_details['parameter']])

                    column_headers_to_parse = column_headers_ind #{} # initialize column headers with indices
                    column_headers_to_parse['start_dt'] = -1

                    self.parsed_data = {column_header:[] for column_header in column_headers_to_parse}
                    self.parsed_data['end_dt'] = []

                #data lines
                else:
                    # skip rows that do not have start date /end date
                    if (row[1] == '') or (row[3] == ''):
                        continue # skip the row
                    else:# parse row
                        for column_header in column_headers_to_parse:
                            if column_header == 'start_dt':
                                    self.parsed_data['start_dt'].append(datetime.datetime.strptime(row[1], '%m/%d/%Y %I:%M:%S %p'))
                                    self.parsed_data['end_dt'].append(datetime.datetime.strptime(row[3], '%m/%d/%Y %I:%M:%S %p'))   
                            else:
                                self.parsed_data[column_header].append(row[column_headers_to_parse[column_header]])

        
        #convert lists to numpy arrays
        for column_header in list(self.parsed_data.keys()):
            if column_header in ['start_dt','end_dt']:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header])
            else:
                self.parsed_data[column_header] = np.array(self.parsed_data[column_header], dtype=np.str_)
            
        #return self.parsed_data