#!/usr/bin/env python
# -*- coding: utf-8 -*-

#WRITTEN BY DENE BOWDALO

###------------------------------------------------------------------------------------###
###IMPORT MODULES
###------------------------------------------------------------------------------------###

import bisect
import __builtin__
import classification_modules
from collections import Counter, defaultdict
import copy
import csv
import datetime
from dateutil.relativedelta import relativedelta
import glob
import itertools
import multiprocessing 
from netCDF4 import Dataset, num2date, date2num, stringtochar, chartostring
import numpy as np
import numpy.lib.recfunctions as rf
import os
import pandas as pd
import pytz
import quality_control_modules
import re
import standardisation_modules
import string
import struct
import subprocess
import time
from timezonefinder import TimezoneFinder
import unicodedata

utc = pytz.utc
timezone_finder_obj = TimezoneFinder()

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###

#dictionary that stores key details and codes for processing each parameter, across all the different networks

def get_parameter_details():

    parameter_details={ 
    'O3':                {'long_parameter_name':'ozone',                           'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'44201', 'airbase_name':'Ozone (air)',                          'airbase_code':'00007', 'ebas_parameter_name':'ozone',                    'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'O3',      'bsc_parameter_name':'sconco3'    },
    'NO':                {'long_parameter_name':'nitrogen monoxide',               'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'42601', 'airbase_name':'Nitrogen monoxide (air)',              'airbase_code':'00038', 'ebas_parameter_name':'nitrogen_monoxide',        'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'NO',      'bsc_parameter_name':'sconcno'    },
    'NO2':               {'long_parameter_name':'nitrogen dioxide',                'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'42602', 'airbase_name':'Nitrogen dioxide (air)',               'airbase_code':'00008', 'ebas_parameter_name':'nitrogen_dioxide',         'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'NO2',     'bsc_parameter_name':'sconcno2'   },
    'CO':                {'long_parameter_name':'carbon monoxide',                 'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':10.1, 'extreme_upper_limit':10000.0, 'aqs_code':'42101', 'airbase_name':'Carbon monoxide (air)',                'airbase_code':'00010', 'ebas_parameter_name':'carbon_monoxide',          'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'CO',      'bsc_parameter_name':'sconcco'    },
    'ISOPRENE':          {'long_parameter_name':'isoprene',                        'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':0.21, 'extreme_upper_limit':200.0,   'aqs_code':'43243', 'airbase_name':'Isoprene (air)',                       'airbase_code':'00451', 'ebas_parameter_name':'isoprene',                 'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'C5H8',    'bsc_parameter_name':'sconcisop'  },
    'SO2':               {'long_parameter_name':'sulphur dioxide',                 'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Sulphur dioxide (air)',                'airbase_code':'00001', 'ebas_parameter_name':'sulphur_dioxide',          'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'SO2',     'bsc_parameter_name':'sconcso2'   },
    'NH3':               {'long_parameter_name':'ammonia',                         'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':0.21, 'extreme_upper_limit':100.0,   'aqs_code':'',      'airbase_name':'Ammonia (air)',                        'airbase_code':'00035', 'ebas_parameter_name':'ammonia',                  'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'NH3',     'bsc_parameter_name':'sconcnh3'   },
    'HNO3':              {'long_parameter_name':'nitric acid',                     'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':0.21, 'extreme_upper_limit':100.0,   'aqs_code':'',      'airbase_name':'Nitric acid (air)',                    'airbase_code':'00050', 'ebas_parameter_name':'nitric_acid',              'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'HNO3',    'bsc_parameter_name':'sconchno3'  },
    'PAN':               {'long_parameter_name':'peroxyacetyl nitrate',            'matrix':'gas',     'standard_units':'nmol mol-1', 'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Peroxyacetyl nitrate (air)',           'airbase_code':'00034', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'air',     'ebas_priority_units':['mol/mol','/m3'], 'chemical_formula':'C2H3NO5', 'bsc_parameter_name':'sconcpan'   },
    'PM10':              {'long_parameter_name':'PM10 mass',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':2000.0,  'aqs_code':'',      'airbase_name':'Particulate matter < 10 µm (aerosol)', 'airbase_code':'00005', 'ebas_parameter_name':'pm10_mass',                'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'',        'bsc_parameter_name':'pm10'       },
    'PM2.5':             {'long_parameter_name':'PM2.5 mass',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':2000.0,  'aqs_code':'',      'airbase_name':'Particulate matter < 2.5 m (aerosol)', 'airbase_code':'06001', 'ebas_parameter_name':'pm25_mass',                'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'',        'bsc_parameter_name':'pm2p5'      },
    'PM1':               {'long_parameter_name':'PM1 mass',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':2000.0,  'aqs_code':'',      'airbase_name':'Particulate matter < 1 µm (aerosol)',  'airbase_code':'06002', 'ebas_parameter_name':'pm1_mass',                 'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'',        'bsc_parameter_name':'pm1'        },
    'Al':                {'long_parameter_name':'aluminium',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'aluminium (aerosol)',                  'airbase_code':'00604', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Al',      'bsc_parameter_name':'sconcal'    },
    'As':                {'long_parameter_name':'arsenic',                         'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Arsenic (aerosol)',                    'airbase_code':'00018', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'As',      'bsc_parameter_name':'sconcas'    },
    'BC':                {'long_parameter_name':'black carbon',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'black_carbon (aerosol)',               'airbase_code':'00391', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'sconcbc'    },
    'C':                 {'long_parameter_name':'carbon',                          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon',             'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'sconcc'     },
    'C_corrected':       {'long_parameter_name':'carbon: corrected',               'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon_corrected',   'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'Ca++':              {'long_parameter_name':'calcium',                         'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'calcium (aerosol)',                    'airbase_code':'00629', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Ca',      'bsc_parameter_name':'sconcca'    },
    'Cd':                {'long_parameter_name':'cadmium',                         'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Cadmium (aerosol)',                    'airbase_code':'00014', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Cd',      'bsc_parameter_name':'sconccd'    },
    'Cl-':               {'long_parameter_name':'chloride',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'chloride (aerosol)',                   'airbase_code':'00631', 'ebas_parameter_name':'chloride',                 'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Cl',      'bsc_parameter_name':'sconccl'    },
    'Co':                {'long_parameter_name':'cobalt',                          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Cobalt (aerosol)',                     'airbase_code':'00064', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Co',      'bsc_parameter_name':'sconccobalt'},
    'Cr':                {'long_parameter_name':'chromium',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Chromium (aerosol)',                   'airbase_code':'00016', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Cr',      'bsc_parameter_name':'sconccr'    },
    'Cu':                {'long_parameter_name':'copper',                          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Copper (aerosol)',                     'airbase_code':'00073', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Cu',      'bsc_parameter_name':'sconccu'    },
    'EC':                {'long_parameter_name':'elemental carbon',                'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Elemental carbon (aerosol)',           'airbase_code':'00771', 'ebas_parameter_name':'elemental_carbon',         'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'sconcec'    },
    'Fe':                {'long_parameter_name':'iron',                            'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Iron (aerosol)',                       'airbase_code':'00065', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Fe',      'bsc_parameter_name':'sconcfe'    },
    'Hg':                {'long_parameter_name':'mercury',                         'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Mercury (aerosol)',                    'airbase_code':'00013', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Hg',      'bsc_parameter_name':'sconchg'    },
    'K+':                {'long_parameter_name':'potassium',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'potassium (aerosol)',                  'airbase_code':'00657', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'K',       'bsc_parameter_name':'sconck'     },
    'Mg++':              {'long_parameter_name':'magnesium',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'magnesium (aerosol)',                  'airbase_code':'00659', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Mg',      'bsc_parameter_name':'sconcmg'    },
    'Mn':                {'long_parameter_name':'manganese',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Manganese (aerosol)',                  'airbase_code':'00017', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Mn',      'bsc_parameter_name':'sconcmn'    }, 
    'Na+':               {'long_parameter_name':'sodium',                          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'sodium (aerosol)',                     'airbase_code':'00668', 'ebas_parameter_name':'sodium',                   'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Na',      'bsc_parameter_name':'sconcna'    },
    'NH4+':              {'long_parameter_name':'ammonium',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Particulate ammonium (aerosol)',       'airbase_code':'00045', 'ebas_parameter_name':'ammonium',                 'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'NH4',     'bsc_parameter_name':'sconcnh4'   },
    'Ni':                {'long_parameter_name':'nickel',                          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Nickel (aerosol)',                     'airbase_code':'00015', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Ni',      'bsc_parameter_name':'sconcni'    },
    'NO3-':              {'long_parameter_name':'nitrate',                         'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Particulate nitrate (aerosol)',        'airbase_code':'00046', 'ebas_parameter_name':'nitrate',                  'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'NO3',     'bsc_parameter_name':'sconcno3'   },
    'OC':                {'long_parameter_name':'organic carbon',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Organic carbon (aerosol)',             'airbase_code':'00772', 'ebas_parameter_name':'organic_carbon',           'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'sconcoc'    },
    'OC_corrected':      {'long_parameter_name':'organic carbon: corrected',       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'organic_carbon_corrected', 'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'Pb':                {'long_parameter_name':'lead',                            'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Lead (aerosol)',                       'airbase_code':'00012', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Pb',      'bsc_parameter_name':'sconcpb'    },
    'Se':                {'long_parameter_name':'selenium',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Selenium (aerosol)',                   'airbase_code':'00048', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Se',      'bsc_parameter_name':'sconcse'    },
    'SO4--':             {'long_parameter_name':'sulphate',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Particulate sulphate (aerosol)',       'airbase_code':'00047', 'ebas_parameter_name':'sulphate_total',           'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'sconcso4'   },
    'SO4--_NSS':         {'long_parameter_name':'sulphate: non-sea salt',          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'sulphate_corrected',       'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'sconcso4nss'},
    'V':                 {'long_parameter_name':'vanadium',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Vanadium (aerosol)',                   'airbase_code':'00049', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'V',       'bsc_parameter_name':'sconcv'     },
    'Zn':                {'long_parameter_name':'zinc',                            'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Zinc (aerosol)',                       'airbase_code':'00063', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'aerosol', 'ebas_priority_units':['/m3'],           'chemical_formula':'Zn',      'bsc_parameter_name':'sconczn'    },
    'PM10_Al':           {'long_parameter_name':'PM10 aluminium',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'aluminium in PM10 (aerosol)',          'airbase_code':'05604', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Al',      'bsc_parameter_name':'pm10al'     },
    'PM10_As':           {'long_parameter_name':'PM10 arsenic',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Arsenic in PM10 (aerosol)',            'airbase_code':'05018', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'As',      'bsc_parameter_name':'pm10as'     },
    'PM10_BC':           {'long_parameter_name':'PM10 black carbon',               'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'black_carbon in PM10 (aerosol)',       'airbase_code':'05391', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm10bc'     },
    'PM10_C':            {'long_parameter_name':'PM10 carbon',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon',             'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm10c'      },
    'PM10_C_corrected':  {'long_parameter_name':'PM10 carbon: corrected',          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon_corrected',   'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'PM10_Ca++':         {'long_parameter_name':'PM10 calcium',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'calcium in PM10 (aerosol)',            'airbase_code':'05629', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Ca',      'bsc_parameter_name':'pm10ca'     },
    'PM10_Cd':           {'long_parameter_name':'PM10 cadmium',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Cadmium in PM10 (aerosol)',            'airbase_code':'05014', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cd',      'bsc_parameter_name':'pm10cd'     },
    'PM10_Cl-':          {'long_parameter_name':'PM10 chloride',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'chloride in PM10 (aerosol)',           'airbase_code':'05631', 'ebas_parameter_name':'chloride',                 'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cl',      'bsc_parameter_name':'pm10cl'     },
    'PM10_Co':           {'long_parameter_name':'PM10 cobalt',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Cobalt in PM10 (aerosol)',             'airbase_code':'05064', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Co',      'bsc_parameter_name':'pm10cobalt' },
    'PM10_Cr':           {'long_parameter_name':'PM10 chromium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Chromium in PM10 (aerosol)',           'airbase_code':'05016', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cr',      'bsc_parameter_name':'pm10cr'     },
    'PM10_Cu':           {'long_parameter_name':'PM10 copper',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Copper in PM10 (aerosol)',             'airbase_code':'05073', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cu',      'bsc_parameter_name':'pm10cu'     },
    'PM10_EC':           {'long_parameter_name':'PM10 elemental carbon',           'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Elemental carbon in PM10 (aerosol)',   'airbase_code':'05771', 'ebas_parameter_name':'elemental_carbon',         'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm10ec'     },
    'PM10_Fe':           {'long_parameter_name':'PM10 iron',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Iron in PM10 (aerosol)',               'airbase_code':'05065', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Fe',      'bsc_parameter_name':'pm10fe'     },
    'PM10_Hg':           {'long_parameter_name':'PM10 mercury',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Mercury in PM10 (aerosol)',            'airbase_code':'05013', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Hg',      'bsc_parameter_name':'pm10hg'     },
    'PM10_K+':           {'long_parameter_name':'PM10 potassium',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'potassium in PM10 (aerosol)',          'airbase_code':'05657', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'K',       'bsc_parameter_name':'pm10k'      },
    'PM10_Mg++':         {'long_parameter_name':'PM10 magnesium',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'magnesium in PM10 (aerosol)',          'airbase_code':'05659', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Mg',      'bsc_parameter_name':'pm10mg'     },
    'PM10_Mn':           {'long_parameter_name':'PM10 manganese',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Manganese in PM10 (aerosol)',          'airbase_code':'05017', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Mn',      'bsc_parameter_name':'pm10mn'     },
    'PM10_Na+':          {'long_parameter_name':'PM10 sodium',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'sodium in PM10 (aerosol)',             'airbase_code':'05668', 'ebas_parameter_name':'sodium',                   'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Na',      'bsc_parameter_name':'pm10na'     },
    'PM10_NH4+':         {'long_parameter_name':'PM10 ammonium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Ammonium in PM10 (aerosol)',           'airbase_code':'05045', 'ebas_parameter_name':'ammonium',                 'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'NH4',     'bsc_parameter_name':'pm10nh4'    },
    'PM10_Ni':           {'long_parameter_name':'PM10 nickel',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Nickel in PM10 (aerosol)',             'airbase_code':'05015', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Ni',      'bsc_parameter_name':'pm10ni'     },
    'PM10_NO3-':         {'long_parameter_name':'PM10 nitrate',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Nitrate in PM10 (aerosol)',            'airbase_code':'05046', 'ebas_parameter_name':'nitrate',                  'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'NO3',     'bsc_parameter_name':'pm10no3'    },
    'PM10_OC':           {'long_parameter_name':'PM10 organic carbon',             'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Organic carbon in PM10 (aerosol)',     'airbase_code':'05772', 'ebas_parameter_name':'organic_carbon',           'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm10oc'     },
    'PM10_OC_corrected': {'long_parameter_name':'PM10 organic carbon: corrected',  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'organic_carbon_corrected', 'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'PM10_Pb':           {'long_parameter_name':'PM10 lead',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Lead in PM10 (aerosol)',               'airbase_code':'05012', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Pb',      'bsc_parameter_name':'pm10pb'     },
    'PM10_Se':           {'long_parameter_name':'PM10 selenium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Selenium in PM10 (aerosol)',           'airbase_code':'05048', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Se',      'bsc_parameter_name':'pm10se'     },
    'PM10_SO4--':        {'long_parameter_name':'PM10 sulphate',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'sulphate in PM10 (aerosol)',           'airbase_code':'05047', 'ebas_parameter_name':'sulphate_total',           'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'pm10so4'    },
    'PM10_SO4--_NSS':    {'long_parameter_name':'PM10 sulphate : non-sea salt',    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'sulphate_corrected',       'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'pm10so4nss' },
    'PM10_V':            {'long_parameter_name':'PM10 vanadium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Vanedium in PM10 (aerosol)',           'airbase_code':'05049', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'V',       'bsc_parameter_name':'pm10v'      },
    'PM10_Zn':           {'long_parameter_name':'PM10 zinc',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Zinc in PM10 (aerosol)',               'airbase_code':'05063', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm10',    'ebas_priority_units':['/m3'],           'chemical_formula':'Zn',      'bsc_parameter_name':'pm10zn'     },
    'PM2.5_Al':          {'long_parameter_name':'PM2.5 aluminium',                 'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'aluminium in PM2.5 (aerosol)',         'airbase_code':'01604', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Al',      'bsc_parameter_name':'pm2p5al'    },
    'PM2.5_As':          {'long_parameter_name':'PM2.5 arsenic',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Arsenic in PM2.5 (aerosol)',           'airbase_code':'01018', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'As',      'bsc_parameter_name':'pm2p5as'    },
    'PM2.5_BC':          {'long_parameter_name':'PM2.5 black carbon',              'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'black_carbon in PM2.5 (aerosol)',      'airbase_code':'01391', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm2p5bc'    },
    'PM2.5_C':           {'long_parameter_name':'PM2.5 carbon',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon',             'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm2p5c'     },
    'PM2.5_C_corrected': {'long_parameter_name':'PM2.5 carbon: corrected',         'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon_corrected',   'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'PM2.5_Ca++':        {'long_parameter_name':'PM2.5 calcium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'calcium in PM2.5 (aerosol)',           'airbase_code':'01629', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Ca',      'bsc_parameter_name':'pm2p5ca'    },
    'PM2.5_Cd':          {'long_parameter_name':'PM2.5 cadmium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Cadmium in PM2.5 (aerosol)',           'airbase_code':'01014', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cd',      'bsc_parameter_name':'pm2p5cd'    },  
    'PM2.5_Cl-':         {'long_parameter_name':'PM2.5 chloride',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'chloride in PM2.5 (aerosol)',          'airbase_code':'01631', 'ebas_parameter_name':'chloride',                 'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cl',      'bsc_parameter_name':'pm2p5cl'    },
    'PM2.5_Co':          {'long_parameter_name':'PM2.5 cobalt',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Cobalt in PM2.5 (aerosol)',            'airbase_code':'01064', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Co',      'bsc_parameter_name':'pm2p5cobalt'},
    'PM2.5_Cr':          {'long_parameter_name':'PM2.5 chromium',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Chromium in PM2.5 (aerosol)',          'airbase_code':'01016', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cr',      'bsc_parameter_name':'pm2p5cr'    },
    'PM2.5_Cu':          {'long_parameter_name':'PM2.5 copper',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Copper in PM2.5 (aerosol)',            'airbase_code':'01073', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Cu',      'bsc_parameter_name':'pm2p5cu'    },
    'PM2.5_EC':          {'long_parameter_name':'PM2.5 elemental carbon',          'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Elemental carbon in PM2.5 (aerosol)',  'airbase_code':'01771', 'ebas_parameter_name':'elemental_carbon',         'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm2p5ec'    },
    'PM2.5_Fe':          {'long_parameter_name':'PM2.5 iron',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Iron in PM2.5 (aerosol)',              'airbase_code':'01065', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Fe',      'bsc_parameter_name':'pm2p5fe'    },
    'PM2.5_Hg':          {'long_parameter_name':'PM2.5 mercury',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Mercury in PM2.5 (aerosol)',           'airbase_code':'01013', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Hg',      'bsc_parameter_name':'pm2p5hg'    },
    'PM2.5_K+':          {'long_parameter_name':'PM2.5 potassium',                 'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'potassium in PM2.5 (aerosol)',         'airbase_code':'01657', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'K',       'bsc_parameter_name':'pm2p5k'     },
    'PM2.5_Mg++':        {'long_parameter_name':'PM2.5 magnesium',                 'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'magnesium in PM2.5 (aerosol)',         'airbase_code':'01659', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Mg',      'bsc_parameter_name':'pm2p5mg'    },
    'PM2.5_Mn':          {'long_parameter_name':'PM2.5 manganese',                 'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Manganese in PM2.5 (aerosol)',         'airbase_code':'01017', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Mn',      'bsc_parameter_name':'pm2p5mn'    },
    'PM2.5_Na+':         {'long_parameter_name':'PM2.5 sodium',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'sodium in PM2.5 (aerosol)',            'airbase_code':'01668', 'ebas_parameter_name':'sodium',                   'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Na',      'bsc_parameter_name':'pm2p5na'    },
    'PM2.5_NH3':         {'long_parameter_name':'PM2.5 ammonia',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'ammonia',                  'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'NH3',     'bsc_parameter_name':''           },
    'PM2.5_NH4+':        {'long_parameter_name':'PM2.5 ammonium',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Ammonium in PM2.5 (aerosol)',          'airbase_code':'01045', 'ebas_parameter_name':'ammonium',                 'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'NH4',     'bsc_parameter_name':'pm2p5nh4'   },
    'PM2.5_Ni':          {'long_parameter_name':'PM2.5 nickel',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Nickel in PM2.5 (aerosol)',            'airbase_code':'01015', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Ni',      'bsc_parameter_name':'pm2p5ni'    },
    'PM2.5_NO3-':        {'long_parameter_name':'PM2.5 nitrate',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Nitrate in PM2.5 (aerosol)',           'airbase_code':'01046', 'ebas_parameter_name':'nitrate',                  'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'NO3',     'bsc_parameter_name':'pm2p5no3'   },
    'PM2.5_OC':          {'long_parameter_name':'PM2.5 organic carbon',            'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Organic carbon in PM2.5 (aerosol)',    'airbase_code':'01772', 'ebas_parameter_name':'organic_carbon',           'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm2p5oc'    },
    'PM2.5_OC_corrected':{'long_parameter_name':'PM2.5 organic carbon: corrected', 'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'organic_carbon_corrected', 'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'PM2.5_Pb':          {'long_parameter_name':'PM2.5 lead',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Lead in PM2.5 (aerosol)',              'airbase_code':'01012', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Pb',      'bsc_parameter_name':'pm2p5pb'    },
    'PM2.5_Se':          {'long_parameter_name':'PM2.5 selenium',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Selenium in PM2.5 (aerosol)',          'airbase_code':'01048', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Se',      'bsc_parameter_name':'pm2p5se'    },
    'PM2.5_SO4--':       {'long_parameter_name':'PM2.5 sulphate',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'sulphate in PM2.5 (aerosol)',          'airbase_code':'01047', 'ebas_parameter_name':'sulphate_total',           'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'pm2p5so4'   },
    'PM2.5_SO4--_NSS':   {'long_parameter_name':'PM2.5 sulphate : non-sea salt',   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'sulphate_corrected',       'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'pm2p5so4nss'},
    'PM2.5_V':           {'long_parameter_name':'PM2.5 vanadium',                  'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Vanedium in PM2.5 (aerosol)',          'airbase_code':'01049', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'V',       'bsc_parameter_name':'pm2p5v'     },
    'PM2.5_Zn':          {'long_parameter_name':'PM2.5 zinc',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'Zinc in PM2.5 (aerosol)',              'airbase_code':'01063', 'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm25',    'ebas_priority_units':['/m3'],           'chemical_formula':'Zn',      'bsc_parameter_name':'pm2p5zn'    },
    'PM1_Al':            {'long_parameter_name':'PM1 aluminium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Al',      'bsc_parameter_name':'pm1al'      },
    'PM1_As':            {'long_parameter_name':'PM1 arsenic',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'As',      'bsc_parameter_name':'pm1as'      },
    'PM1_BC':            {'long_parameter_name':'PM1 black carbon',                'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm1bc'      },
    'PM1_C':             {'long_parameter_name':'PM1 carbon',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon',             'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm1c'       },
    'PM1_C_corrected':   {'long_parameter_name':'PM1 carbon: corrected',           'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'total_carbon_corrected',   'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'PM1_Ca++':          {'long_parameter_name':'PM1 calcium',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Ca',      'bsc_parameter_name':'pm1ca'      },
    'PM1_Cd':            {'long_parameter_name':'PM1 cadmium',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Cd',      'bsc_parameter_name':'pm1cd'      },
    'PM1_Cl-':           {'long_parameter_name':'PM1 chloride',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'chloride',                 'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Cl',      'bsc_parameter_name':'pm1cl'      },
    'PM1_Co':            {'long_parameter_name':'PM1 cobalt',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Co',      'bsc_parameter_name':'pm1cobalt'  },
    'PM1_Cr':            {'long_parameter_name':'PM1 chromium',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Cr',      'bsc_parameter_name':'pm1cr'      },
    'PM1_Cu':            {'long_parameter_name':'PM1 copper',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Cu',      'bsc_parameter_name':'pm1cu'      },
    'PM1_EC':            {'long_parameter_name':'PM1 elemental carbon',            'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'elemental_carbon',         'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm1ec'      },
    'PM1_Fe':            {'long_parameter_name':'PM1 iron',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Fe',      'bsc_parameter_name':'pm1fe'      },
    'PM1_Hg':            {'long_parameter_name':'PM1 mercury',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Hg',      'bsc_parameter_name':'pm1hg'      },
    'PM1_K+':            {'long_parameter_name':'PM1 potassium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'K',       'bsc_parameter_name':'pm1k'       },
    'PM1_Mg++':          {'long_parameter_name':'PM1 magnesium',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Mg',      'bsc_parameter_name':'pm1mg'      },
    'PM1_Mn':            {'long_parameter_name':'PM1 manganese',                   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Mn',      'bsc_parameter_name':'pm1mn'      },
    'PM1_Na+':           {'long_parameter_name':'PM1 sodium',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'sodium',                   'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Na',      'bsc_parameter_name':'pm1na'      },
    'PM1_NH4+':          {'long_parameter_name':'PM1 ammonium',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'ammonium',                 'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'NH4',     'bsc_parameter_name':'pm1nh4'     },
    'PM1_Ni':            {'long_parameter_name':'PM1 nickel',                      'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Ni',      'bsc_parameter_name':'pm1ni'      },
    'PM1_NO3-':          {'long_parameter_name':'PM1 nitrate',                     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'nitrate',                  'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'NO3',     'bsc_parameter_name':'pm1no3'     },
    'PM1_OC':            {'long_parameter_name':'PM1 organic carbon',              'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'organic_carbon',           'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':'pm1oc'      },
    'PM1_OC_corrected':  {'long_parameter_name':'PM1 organic carbon: corrected',   'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'organic_carbon_corrected', 'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'C',       'bsc_parameter_name':''           },
    'PM1_Pb':            {'long_parameter_name':'PM1 lead',                        'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Pb',      'bsc_parameter_name':'pm1pb'      },
    'PM1_Se':            {'long_parameter_name':'PM1 selenium',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Se',      'bsc_parameter_name':'pm1se'      },
    'PM1_SO4--':         {'long_parameter_name':'PM1 sulphate',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'sulphate_total',           'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'pm1so4'     },
    'PM1_SO4--_NSS':     {'long_parameter_name':'PM1 sulphate : non-sea salt',     'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'sulphate_corrected',       'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'SO4',     'bsc_parameter_name':'pm1so4nss'  },
    'PM1_V':             {'long_parameter_name':'PM1 vanadium',                    'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'V',       'bsc_parameter_name':'pm1v'       },
    'PM1_Zn':            {'long_parameter_name':'PM10 zinc',                       'matrix':'aerosol', 'standard_units':'ug m-3',     'minimum_resolution':1.1,  'extreme_upper_limit':1000.0,  'aqs_code':'',      'airbase_name':'',                                     'airbase_code':'',      'ebas_parameter_name':'NaN',                      'ebas_matrix':'pm1',     'ebas_priority_units':['/m3'],           'chemical_formula':'Zn',      'bsc_parameter_name':'pm1zn'      }}
    
    return parameter_details

###------------------------------------------------------------------------------------###

#dictionary that stores key details for processing data from the different networks

def get_network_details():

    network_details={ 
    'AIRBASE':{'file_storage_type':'site_per_year',       'external_metadata_file':'',                     'metadata_data_join_variable_name':''},
    'AQS':    {'file_storage_type':'all_per_year',        'external_metadata_file':'',                     'metadata_data_join_variable_name':''},
    'CAPMON': {'file_storage_type':'all_per_year',        'external_metadata_file':'',                     'metadata_data_join_variable_name':''},
    'CASTNET':{'file_storage_type':'all_per_year',        'external_metadata_file':'',                     'metadata_data_join_variable_name':''},
    'EANET':  {'file_storage_type':'site_per_year',       'external_metadata_file':'',                     'metadata_data_join_variable_name':''},
    'EBAS':   {'file_storage_type':'site_per_multiyear',  'external_metadata_file':'EBAS_MANUAL_META.csv', 'metadata_data_join_variable_name':'reference'},
    'EIONET': {'file_storage_type':'site_per_year',       'external_metadata_file':'EIONET_META.csv',      'metadata_data_join_variable_name':''},
    'GAW':    {'file_storage_type':'site_per_year',       'external_metadata_file':'',                     'metadata_data_join_variable_name':''},
    'NAPS':   {'file_storage_type':'all_per_year',        'external_metadata_file':'',                     'metadata_data_join_variable_name':''},
    'SEARCH': {'file_storage_type':'all_per_year',        'external_metadata_file':'',                     'metadata_data_join_variable_name':''}}
    
    return network_details


###------------------------------------------------------------------------------------###
    
#gets list of all relevant data files for network/parameter combination
    
def get_data_files():
    
    if run_details['network'] == 'EBAS':
        #special case for NO2 (due to NOx being given in joint files - all of which are stored under NO)
        if run_details['parameter'] == 'NO2':
            nox_files = glob.glob('/esarchive/obs/ghost/EBAS/original_files/gas/NO/*.nas')
            data_files = glob.glob('/esarchive/obs/ghost/EBAS/original_files/gas/%s/*.nas'%(run_details['parameter']))
            #append files in NO directory which contain NO2
            for f in nox_files:
                if 'nitrogen_monoxide' not in f:
                    data_files.append(f)
        #handle complicated aerosol parameters 
        #PM10, PM2.5, PM1
        elif (run_details['parameter'] == 'PM10') or (run_details['parameter'] == 'PM2.5') or (run_details['parameter'] == 'PM1'):
            data_files = glob.glob('/esarchive/obs/ghost/%s/original_files/aerosol/%s/mass/*.nas'%(run_details['network'], run_details['parameter']))
        #bulk speciated
        elif (run_details['parameter'] == 'C') or (run_details['parameter'] == 'C_corrected') or (run_details['parameter'] == 'Cl-') or (run_details['parameter'] == 'EC') or (run_details['parameter'] == 'Na+') or (run_details['parameter'] == 'NH4+') or (run_details['parameter'] == 'NO3-') or (run_details['parameter'] == 'OC') or (run_details['parameter'] == 'OC_corrected') or (run_details['parameter'] == 'SO4--') or (run_details['parameter'] == 'SO4--_NSS'):
            data_files = glob.glob('/esarchive/obs/ghost/%s/original_files/aerosol/bulk/speciated/%s/*.nas'%(run_details['network'], run_details['parameter']))
        #PM10/PM2.5/PM1 speciated
        elif ('PM10_' in run_details['parameter']) or ('PM2.5_' in run_details['parameter']) or ('PM1_' in run_details['parameter']):
            aerosol_type = run_details['parameter'].split('_')[0]
            param = run_details['parameter'].split('_')[-1]
            data_files = glob.glob('/esarchive/obs/ghost/%s/original_files/aerosol/%s/speciated/%s/*.nas'%(run_details['network'], aerosol_type, param))        
        #all other parameters
        else:
            data_files = glob.glob('/esarchive/obs/ghost/%s/original_files/%s/%s/*.nas'%(run_details['network'],parameter_details['matrix'],run_details['parameter']))

    if run_details['network'] == 'EIONET':
        #handle complicated aerosol parameters 
        #PM10, PM2.5, PM1
        if (run_details['parameter'] == 'PM10') or (run_details['parameter'] == 'PM2.5') or (run_details['parameter'] == 'PM1'):
            stem = '/esarchive/obs/ghost/%s/original_files/aerosol/%s/mass'%(run_details['network'], run_details['parameter'])
        #bulk speciated
        elif (run_details['parameter'] == 'Al') or (run_details['parameter'] == 'As') or (run_details['parameter'] == 'BC') or (run_details['parameter'] == 'Ca++') or (run_details['parameter'] == 'Cd') or (run_details['parameter'] == 'Cl-') or (run_details['parameter'] == 'Co') or (run_details['parameter'] == 'Cr') or (run_details['parameter'] == 'Cu') or (run_details['parameter'] == 'EC') or (run_details['parameter'] == 'Fe') or (run_details['parameter'] == 'Hg') or (run_details['parameter'] == 'K+') or (run_details['parameter'] == 'Mg++') or (run_details['parameter'] == 'Mn') or (run_details['parameter'] == 'Na+') or (run_details['parameter'] == 'NH4+') or (run_details['parameter'] == 'Ni') or (run_details['parameter'] == 'NO3-') or (run_details['parameter'] == 'OC') or (run_details['parameter'] == 'Pb') or (run_details['parameter'] == 'Se') or (run_details['parameter'] == 'SO4--') or (run_details['parameter'] == 'V') or (run_details['parameter'] == 'Zn'):
            stem = '/esarchive/obs/ghost/%s/original_files/aerosol/bulk/speciated/%s'%(run_details['network'], run_details['parameter'])
        #PM10/PM2.5/PM1 speciated
        elif ('PM10_' in run_details['parameter']) or ('PM2.5_' in run_details['parameter']) or ('PM1_' in run_details['parameter']):
            aerosol_type = run_details['parameter'].split('_')[0]
            param = run_details['parameter'].split('_')[-1]
            stem = '/esarchive/obs/ghost/%s/original_files/aerosol/%s/speciated/%s'%(run_details['network'], aerosol_type, param)        
        #all other parameters
        else:
            stem = '/esarchive/obs/ghost/%s/original_files/%s/%s'%(run_details['network'],parameter_details['matrix'],run_details['parameter'])
        #gather data files
        data_files = []
        for root, dirs, files in os.walk(stem):
            for file in files:
                if file.endswith(".csv"):
                    data_files.append(os.path.join(root, file))

    return data_files
    
###------------------------------------------------------------------------------------###
#generate list of lists splitting out data files relevant to the different stations

def generate_data_files_LoL(station_references,data_files):
    
    data_files_LoL = [[] for _ in xrange(len(station_references))]
    
    for ref_i,ref in enumerate(station_references):
        for data_file in data_files:
            if ref in data_file:
                data_files_LoL[ref_i].append(data_file)

    return data_files_LoL

###------------------------------------------------------------------------------------###

#get unique list of station references provided in data filenames.
#process for this differs by network.

def get_station_references(data_files):
    #reference_codes = {'station_reference':[],'metadata_data_join_variable':[]}
    station_references = []
    
    if run_details['network'] == 'EBAS':    
        for i, f in enumerate(data_files):
            f = f.split('/')[-1]
            
            #reference_codes['station_reference'].append(f.split('.')[0])
            station_references.append(f.split('.')[0])
            
    if run_details['network'] == 'EIONET':
        for i, f in enumerate(data_files):
            f_split = f.split('/')[-1]
            f_split = f_split.split('_')
            
            #reference_codes['station_reference'].append('%s_%s_%s'%(f_split[0],f_split[1],f_split[2]))
            station_references.append('%s_%s_%s_'%(f_split[0],f_split[1],f_split[2]))
        
    #iterate through all keys, and then sort variables in a human-intuitive order
    #for var_name in reference_codes.keys():
    #    reference_codes[var_name] = natsorted(remove_duplicates_preserve_order(reference_codes[var_name]))
    station_references = natsorted(remove_duplicates_preserve_order(station_references))
           
    return station_references

###------------------------------------------------------------------------------------###

#read external metadata file into memory and where possible, cut it for specific stations

def read_external_metadata(station_references):
    
    external_metadata_file = '/esarchive/obs/ghost/%s/metadata/network_provided/%s'%(run_details['network'],network_details['external_metadata_file'])
        
    with open(external_metadata_file, 'rb') as f:
        reader = csv.reader(f,delimiter=',')
        for row_i, row in enumerate(reader):
            if row_i == 0:
                meta_variable_names = row
                external_metadata = [[] for _ in xrange(len(meta_variable_names))]
            else: 
                for meta_variable_name in meta_variable_names:
                    external_metadata[meta_variable_names.index(meta_variable_name)].append(row[meta_variable_names.index(meta_variable_name)].strip())
                
        #convert to numpy record array, to allow indexing with column names
        external_metadata = np.rec.fromarrays(external_metadata, names=meta_variable_names)
        
    #as an exception load EIONET metadata all into memory as no way to make join between data and metadata without opening data files 
    #for EIONET this means loading 3 files into memory: 
    if run_details['network'] == 'EIONET':
        #make standard EIONET metadata file accesible globally
        __builtin__.external_metadata = external_metadata
        
        #read extra metadata AIDE D file
        EIONET_AIDE_D_metadata_file = '/esarchive/obs/ghost/%s/metadata/network_provided/EIONET_AIDE_D.csv'%(run_details['network'])
        with open(EIONET_AIDE_D_metadata_file, 'rb') as f:
            reader = csv.reader(f,delimiter=',')
            for row_i, row in enumerate(reader):
                if row_i == 0:
                    meta_variable_names = row
                    EIONET_AIDE_D_metadata = [[] for _ in xrange(len(meta_variable_names))]
                else: 
                    for meta_variable_name in meta_variable_names:
                        EIONET_AIDE_D_metadata[meta_variable_names.index(meta_variable_name)].append(row[meta_variable_names.index(meta_variable_name)].strip())
                                        
        #convert to numpy record array, to allow indexing with column names
        EIONET_AIDE_D_metadata = np.rec.fromarrays(EIONET_AIDE_D_metadata, names=meta_variable_names)
        #make AIDE D metadata accesible globally
        __builtin__.external_metadata_AIDE_D = EIONET_AIDE_D_metadata
        
        #read AIRBASE v8 processed .npy file
        AIRBASE_v8_metadata_file = '/esarchive/obs/ghost/AIRBASE/metadata/processed/AIRBASE_META.npy'
        AIRBASE_v8_metadata = np.load(AIRBASE_v8_metadata_file) 
        #make AIRBASE v8 metadata accesible globally
        __builtin__.external_metadata_AIRBASE = AIRBASE_v8_metadata
        
        #return empty array (so processing doesn't break)
        empty_external_metadata = [[] for _ in xrange(len(station_references))]
        return empty_external_metadata
    
    #cut external meta array to just be those for references you have in data files
    external_metadata_valid_inds = np.empty(0, dtype=np.int)
    do_not_have_reference_in_meta = []
    for ref in station_references:
        try:
            external_metadata_valid_inds=np.append(external_metadata_valid_inds, np.where(external_metadata[network_details['metadata_data_join_variable_name']] == ref)[0][0])
        except:
            do_not_have_reference_in_meta.append(ref) 
   
   #if some references from data files are not in the external metadata array, print to screen and stop processing
    if len(do_not_have_reference_in_meta) > 0:
        print 'DO NOT HAVE THESE REFERENCES IN EXTERNAL STATION METADATA:'
        print do_not_have_reference_in_meta
        1+'a'
   
    #cut external meta array
    external_metadata = external_metadata[external_metadata_valid_inds]
    
    return external_metadata
    
###------------------------------------------------------------------------------------###

#process nasa ames files 

#only requires filename input

class process_nasa_ames(object):
    
    def __init__(self,filename): 
        self.filename = filename
        
        #core header meta
        self.n_header = ''          #number of header lines 
        self.ffi = ''               #file format index 
                                    #1st digit indicates the number of independent variables (dimensions)
                                    #2nd digit indicates number of necessary auxiliary variable 
                                    #Remaining differences cover the way the independent variables increment and the nature (numeric or character) of the variables
        self.originator_name = ''   #data originator name
        self.originator_affil = ''  #data originator affiliation
        self.provider_name = ''     #data provider name 
        self.campaign_name = ''     #campaign name
        self.volume_number = ''     #current volume number (dated field - should be 1)
        self.n_volumes = ''         #number of volumes in dataset (dated field - should be 1)
        self.date_start = ''        #date for which record starts
        self.file_creation = ''     #date of file creation 
    
    
        #independent variable meta (e.g. time)
        self.independent_interval = ''  #independent variable interval (if changes should be zero)
        self.independent_name = ''      #independent variable name (e.g. days from file reference point) 
    
    
        #primary variable meta (quantities that have been observed or derived)
        self.n_primary = ''               #number of primary variables
        self.primary_scale_factors = ''   #scale factors for each of primary variables
        self.primary_missing_values = ''  #value imposed for missing data for each primary variable  
        self.primary_names = []           #primary variable names (e.g. ozone)
        self.primary_units = []           #primary variable units
        self.primary_details = []         #extra details about variables (e.g. limit of detection)
    
        #auxiliary variable meta
        self.n_auxiliary = ''    #number of auxiliary variables
    
        #comments meta
        self.n_comments = ''     #number of lines of comments
        self.comments = []       #extra meta comments about data. Can also include column headings  
    
        #important line counts
        self.endline_primary_count = -999
        self.start_auxiliary = -999
        self.start_comments = -999
        self.start_data = -999
        self.row_count = 0
        
    def read_header(self):
    
        with open(self.filename) as f:
            self.content = f.readlines()
            self.content = [x.strip('\n') for x in self.content]
            
            for row in self.content: 
                
                #--read core header meta--
                if self.row_count == 0: 
                    line_split = row.split(' ')
                    self.n_header,self.ffi = line_split[0],line_split[1]
                    
                elif self.row_count == 1:
                    self.originator_name = row      
                    
                elif self.row_count == 2:
                    self.originator_affil = row  
                    
                elif self.row_count == 3:
                    self.provider_name = row 
                    
                elif self.row_count == 4:
                    self.campaign_name = row 
                    
                elif self.row_count == 5:
                    line_split = row.split(' ')
                    self.volume_name, self.n_volumes = line_split[0],line_split[1]
                
                elif self.row_count == 6:
                    line_split = row.split(' ')
                    self.date_start, self.file_creation = line_split[0]+line_split[1]+line_split[2], line_split[3]+line_split[4]+line_split[5]
                    
                #--read independent variable meta--
                elif self.row_count == 7:
                    self.independent_interval = row
                    
                elif self.row_count == 8:
                    self.independent_name = row
                    
                #--read primary variable meta--
                elif self.row_count == 9:
                    self.n_primary = int(row)
                    self.endline_primary_count = 11+self.n_primary
                    self.start_auxiliary = self.endline_primary_count+1
                    
                elif self.row_count == 10:
                    self.primary_scale_factors = row.split(' ')
                    
                elif self.row_count == 11:
                    self.primary_missing_values = row.split(' ')
                    
                elif (self.row_count > 11) & (self.row_count <= self.endline_primary_count):
                    
                    #handle line splitting with quotes (with commas inside)
                    if '"' in row:
                        partition = row.partition('"')
                        comments = partition[-1][:-1]
                        before_comments = partition[0]
                        if len(before_comments) > 0:
                            line_split = before_comments[:-1].split(',')
                            line_split.append(comments)
                        else:
                            line_split = [comments]
                    #line split without quotes    
                    else:
                        line_split = row.split(',')
                    
                    #only read line if first value is one of accepted primary variable names
                    #otherwise line is appended to primary details - this deals with badly formatted files
                    #also add 1 to auxiliary start count - so don't start reading too soon
                    
                    #create a list of all potentially valid primary variables
                    core_parameter_dict = get_parameter_details()
                    valid_primary_variables = ['end_time of measurement','numflag']
                    for param in core_parameter_dict.keys():
                        valid_primary_variables=np.append(valid_primary_variables,core_parameter_dict[param]['ebas_parameter_name'])
                        #create numflag variable for all parameter
                        valid_primary_variables=np.append(valid_primary_variables,'numflag %s'%(core_parameter_dict[param]['ebas_parameter_name']))

                    if line_split[0] in valid_primary_variables: 
                        self.primary_names.append(line_split[0])
                        self.primary_units.append(line_split[1])
                        if len(line_split) > 2:
                            self.primary_details.append(line_split[2:])
                        else:
                            self.primary_details.append([])
                    else:                      
                        self.primary_details[-1] = self.primary_details[-1]+line_split
                        self.endline_primary_count+=1
                        self.start_auxiliary+=1
                    
                #--read auxiliary variable meta--
                #first checks if row is a valid number or relates to the previous primary details not correctly formatted
                #if so then append to primary details  
                elif self.row_count == self.start_auxiliary:
                    try:
                        self.n_auxiliary = int(row)
                        #if is not 0 then break processing and panic - as have never seen this be used and don't know format
                        if self.n_auxiliary != 0:
                            print 'Using auxilary variables. %s of them! How Strange. Now need to code for this. Stopping processing.'%(self.n_auxiliary)
                            print self.filename
                            1+'a'
                        self.start_comments = self.start_auxiliary+1
                    except:
                        line_split = row.split(',')
                        self.primary_details[-1] = np.append(self.primary_details[-1],line_split)
                        self.endline_primary_count+=1
                        self.start_auxiliary+=1

                #--read comments meta--
                elif self.row_count == self.start_comments:
                    self.n_comments = int(row)
                    self.end_comments = self.start_comments+self.n_comments
                    self.start_data = self.end_comments+1
                    
                elif (self.row_count > self.start_comments) & (self.row_count <= self.end_comments):
                    self.comments.append(row)
                    
                #manually iterate row count
                self.row_count+=1
    
    def read_data(self):
        #cut file read to just be data columns
        data = self.content[self.start_data:]
        
        #make list of lists to store data columns (1 independent variable + primary variables)
        self.structured_data = [[] for _ in xrange(self.n_primary+1)]
        
        #iterate through lines, appending column data into a list of lists 
        for line in data:
            line_split = line.split()
            for n,var in enumerate(line_split):
                self.structured_data[n].append(var)
        

###------------------------------------------------------------------------------------###
#get molecular mass of a specific molecule

def get_molecular_mass(species):
    #atomic masses in g mol-1
    atomic_masses = {
    "H":1.00794,"He":4.002602,"Li":6.941,"Be":9.012182,"B":10.811,"C":12.011,"N":14.00674,"O":15.9994,"F":18.9984032,"Ne":20.1797,"Na":22.989768,"Mg":24.3050,"Al":26.981539,"Si":28.0855,"P":30.973762,"S":32.066,"Cl":35.4527,"Ar":39.948,
    "K":39.0983,"Ca":40.078,"Sc":44.955910,"Ti":47.88,"V":50.9415,"Cr":51.9961,"Mn":54.93805,"Fe":55.847,"Co":58.93320,"Ni":58.6934,"Cu":63.546,"Zn":65.39,"Ga":69.723,"Ge":72.61,"As":74.92159,"Se":78.96,"Br":79.904,"Kr":83.80,
    "Rb":85.4678,"Sr":87.62,"Y":88.90585,"Zr":91.224,"Nb":92.90638,"Mo":95.94,"Tc":98,"Ru":101.07,"Rh":102.90550,"Pd":106.42,"Ag":107.8682,"Cd":112.411,"In":114.82,"Sn":118.710,"Sb":121.757,"Te":127.60,"I":126.90447,"Xe":131.29,
    "Cs":132.90543,"Ba":137.327,"La":138.9055,"Ce":140.115,"Pr":140.90765,"Nd":144.24,"Pm":145,"Sm":150.36,"Eu":151.965,"Gd":157.25,"Tb":158.92534,"Dy":162.50,"Ho":164.93032,"Er":167.26,"Tm":168.93421,"Yb":173.04,"Lu":174.967,
    "Hf":178.49,"Ta":180.9479,"W":183.85,"Re":186.207,"Os":190.2,"Ir":192.22,"Pt":195.08,"Au":196.96654,"Hg":200.59,"Tl":204.3833,"Pb":207.2,"Bi":208.98037,"Po":209,"At":210,"Rn":222,"Fr":223,"Ra":226.0254,"Ac":227,"Th":232.0381,
    "Pa":213.0359,"U":238.0289,"Np":237.0482,"Pu":244,"Am":243,"Cm":247,"Bk":247,"Cf":251,"Es":252,"Fm":257,"Md":258,"No":259,"Lr":260,"Rf":261,"Db":262,"Sg":263,"Bh":262,"Hs":265,"Mt":266}
    
    s = re.findall('([A-Z][a-z]?)([0-9]*)', species)
    compoundweight = 0

    for element, count in s:
        count = int(count or '1')
        compoundweight += atomic_masses[element] * count
    
    #convert to kg mol-1
    compoundweight = compoundweight/1e3
    
    return compoundweight
    
###------------------------------------------------------------------------------------###
#get integer number of atoms of a desired element in a molecule. 
def get_N_atoms(molecule, element):
    parse_list = re.findall(r'[A-Z][a-z]*|\d+', re.sub('[A-Z][a-z]*(?![\da-z])', r'\g<0>1', molecule))
    try:
        n_atoms_element = parse_list[parse_list.index(element)+1]
    except:
        n_atoms_element = 1
    
    return int(n_atoms_element)
    
###------------------------------------------------------------------------------------###
#Module for finding and returning values/indices of all duplicates in a string/list 
#Returns list of tuples. 
#The first value in each tuple is the duplicated value
#The second value in each tuple is a list of duplicated indices
#Returns empty list if no duplicates are found
#Module is optimised for speed

def list_duplicates(seq):
    tally = defaultdict(list)
    for i,item in enumerate(seq):
        tally[item].append(i)
    return sorted(((key,locs) for key,locs in tally.items() 
                            if len(locs)>1))

###------------------------------------------------------------------------------------###

#rounds datetime object to the nearest minute

def round_to_minute(dt):
    dt_start_of_minute = dt.replace(second=0, microsecond=0)
    dt_half_minute = dt.replace(second=30, microsecond=0)

    if dt >= dt_half_minute:
        # round up
        dt = dt_start_of_minute + datetime.timedelta(minutes=1)
    else:
        # round down
        dt = dt_start_of_minute

    return dt

#rounds datetime object to the nearest hour

def round_to_hour(dt):
    dt_start_of_hour = dt.replace(minute=0, second=0, microsecond=0)
    dt_half_hour = dt.replace(minute=30, second=0, microsecond=0)

    if dt >= dt_half_hour:
        # round up
        dt = dt_start_of_hour + datetime.timedelta(hours=1)
    else:
        # round down
        dt = dt_start_of_hour

    return dt 
    
###------------------------------------------------------------------------------------###

#modules to interconvert between datetimes and fractional times 

def datetime_to_fractional_days_from_reference(dateFuture):
    #get reference start datetime
    datePast = datetime.datetime(0001,01,01,00,00)
    difference = dateFuture - datePast
    return difference.total_seconds() / datetime.timedelta(days=1).total_seconds()
    
def datetime_to_fractional_minutes_from_reference(dateFuture):
    #get reference start datetime
    datePast = datetime.datetime(0001,01,01,00,00)
    difference = dateFuture - datePast
    return difference.total_seconds() / datetime.timedelta(minutes=1).total_seconds()

def fractional_days_from_reference_to_datetime(dateFuture):
    #get reference start datetime
    datePast = datetime.datetime(0001,01,01,00,00)
    future_timedelta = datetime.timedelta(days=dateFuture)
    return datePast + future_timedelta
    
def fractional_minutes_from_reference_to_datetime(dateFuture):
    #get reference start datetime
    datePast = datetime.datetime(0001,01,01,00,00)
    future_timedelta = datetime.timedelta(minutes=dateFuture)
    return datePast + future_timedelta

###------------------------------------------------------------------------------------###
#remove duplicates from list while preserving order 

def remove_duplicates_preserve_order(seq):
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

###------------------------------------------------------------------------------------###
#submit slurm script from python

def submit_slurm_from_python(n_processes, jobname, t0):
    cmd = ['sbatch', '-n', str(n_processes), '%s_submit.sh'%(jobname)]

    #submit stage to slurm 
    with open(os.devnull, 'w') as devnull:
        subprocess.Popen(cmd, cwd='/esarchive/obs/ghost/process/run/slurm_submit', stdout=devnull)
    #pause for 5 seconds to give system time to register job
    time.sleep(5)
    
    #only continue once job is complete
    #proceed once jobname is no longer in slurm queue
    while True:
        process = subprocess.Popen(["squeue", "-u", "dbowdalo"], stdout=subprocess.PIPE)
        job_status = process.communicate()[0]
        if jobname not in job_status:
            break
        else:
            time.sleep(5)
            
    #check if job completed successfully or not
    line = subprocess.check_output(['tail', '-1', '/esarchive/obs/ghost/process/run/slurm_logs/%s.out'%(jobname)]).strip()
    if line == '%s DONE'%(jobname):
        print '%s PROCESSING COMPLETE IN %s MINUTES'%(jobname,(time.time()-t0)/60.)
    else:
        print '%s DID NOT FINISH. TERMINATING GHOST PIPELINE HERE'%(jobname)
        1+'a'
    
    return 
    
###------------------------------------------------------------------------------------###
#run MPI4PY interactively

def run_mpi4py_interactively(n_processes, jobname, t0):       
    cmd = ['mpiexec', '-n', str(n_processes), 'python', 'stages/%s.py'%(jobname)]
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    while True:
        output = process.stdout.readline()
        if output == '' and process.poll() is not None:
            break
        if output:
            print output.strip()
    print "%s PROCESSING COMPLETE IN %s MINUTES"%(jobname, (time.time()-t0)/60.)
    
    return

###------------------------------------------------------------------------------------###

#get pytz string timezone, and local timezone offset (in hours) of lat/lon pairing, relative to a reference UTC datetime
#if you give the module a pytz string timezone, it skips the calculation of the pytz string timezone by lat/lon (very costly) and proceeds to calculate local timezone offset
def local_offset_from_utc(reference_utc_dt,latitude=-999.0,longitude=-999.0,pytz_local_tz=None):
    
    #if are not provided 
    if pytz_local_tz == None:
    
        latitude, longitude = float(latitude), float(longitude)
    
        #first try get local timezone within defined timezone polygons in timezonefinder module
        #all polygons (with shortcuts in that area) are checked until one polygon is matched. 
        #None is returned without any match.
        #NOTE: The timezone polygons do NOT follow the shorelines any more. Just because you do not get None, the point could still lie off land!
    
        local_tz = timezone_finder_obj.certain_timezone_at(lat=latitude,lng=longitude)
    
        #try get pytz timezone object from local timezone string

        if local_tz != None:
            try:
                pytz_local_tz = pytz.timezone(local_tz)
            except:
                local_tz = None
    
        #if don't have local_tz or pytz has had a fit and will not recognise a timezone
        #force it to find different timezones until it recognises one
        #this is done using module that finds closest local timezone.
        #simply computes and compares the distances to the polygon boundaries!. 
        #This by default returns the closest timezone of all polygons within +-1 degree lng and +-1 degree lat, or returns None if no matches.
        #Optionally can increase radius search using the delta_degree option.
        #Iteratively increase by 1 deg. until have a match
        #NOTE: force_evaluation=True, is set as some polygons might not be tested (for example when a zone is found to be the closest already), and this prevents this
    
        if local_tz == None:
            current_radius = 1    
        
            while pytz_local_tz == None:
                local_tz = timezone_finder_obj.closest_timezone_at(lat=latitude,lng=longitude,delta_degree=current_radius,force_evaluation=True)
    
                #get pytz timezone object from local timezone string
                #occasionally pytz will have a fit and not recognise a timezone
                #force it to run until it recognises one
                if local_tz != None:
                    try:
                        pytz_local_tz = pytz.timezone(local_tz)
                    except:
                        pass        
                current_radius+=1
    
    #reference utc pytz datetime  
    reference_pytz_dt = utc.localize(reference_utc_dt)
    
    #local time pytz datetime (relative to reference UTC datetime)
    local_pytz_dt = pytz_local_tz.localize(reference_utc_dt)
    
    return pytz_local_tz, (reference_pytz_dt - local_pytz_dt).total_seconds() / 60. / 60.

###------------------------------------------------------------------------------------###

#collection of modules that naturally sort (human intuitively)
#natsorted is module to call to run sort
    
def try_int(s):
    "Convert to integer if possible."
    try: return int(s)
    except: return s

def natsort_key(s):
    "Used internally to get a tuple by which s is sorted."
    return map(try_int, re.findall(r'(\d+|\D+)', s))

def natcmp(a, b):
    "Natural string comparison, case sensitive."
    return cmp(natsort_key(a), natsort_key(b))

def natcasecmp(a, b):
    "Natural string comparison, ignores case."
    return natcmp(a.lower(), b.lower())

def natsort(seq, cmp=natcmp):
    "In-place natural string sort."
    seq.sort(cmp)
    
def natsorted(seq, cmp=natcmp):
    "Returns a copy of seq, sorted by natural string sort."
    temp = copy.copy(seq)
    natsort(temp, cmp)
    return temp

###------------------------------------------------------------------------------------###

#module that strips accents from inout string

def remove_accents(text):
    """
    Strip accents from input String.

    :param text: The input string.
    :type text: String.

    :returns: The processed String.
    :rtype: String.
    """
    try:
        text = unicode(text, 'utf-8')
    except (TypeError, NameError): # unicode is a default on python 3 
        pass
    text = unicodedata.normalize('NFD', text)
    text = text.encode('ascii', 'ignore')
    text = text.decode("utf-8")
    return str(text)
    
###------------------------------------------------------------------------------------###

#get processed temporary station references from files saved out in the temporary/native file directory

def get_processed_temporary_references():
    processed_temporary_references = []

    station_temporary_files = glob.glob('/esarchive/obs/ghost/%s/native/%s/temporary/*.npz'%(run_details['network'],parameter_details['bsc_parameter_name']))
    
    for file_i, file in enumerate(station_temporary_files):
        processed_temporary_references.append(file.split('/')[-1].split('.npz')[0])
    
    #sort references in alphabetical order
    processed_temporary_references = natsorted(processed_temporary_references)
    
    return processed_temporary_references

###------------------------------------------------------------------------------------###

#process non-network specific metadata

def process_non_network_specific_metadata(site_metadata, data_file):

    #get station_timezone (at reference 2000-01-01 00:00 UTC) & wmo_region ,only if have valid latitude and longitude
    if (pd.isnull(site_metadata['latitude']['value'][-1]) == False) & (pd.isnull(site_metadata['longitude']['value'][-1]) == False):
        dt_ref = datetime.datetime(2000,01,01,00,00)
        #call to get timezone is costly, therefore take shortcut, if have already processed this for the station, and lat and lon have not changed, copy from previous iteration, otherwise process it
        if len(site_metadata['station_timezone']['value']) != 1:
            if (site_metadata['latitude']['value'][-1] == site_metadata['latitude']['value'][-2]) & (site_metadata['longitude']['value'][-1] == site_metadata['longitude']['value'][-2]):
                site_metadata['station_timezone']['value'][-1] = site_metadata['station_timezone']['value'][-2]
            else:
                pytz_str, site_metadata['station_timezone']['value'][-1] = local_offset_from_utc(dt_ref,longitude=site_metadata['longitude']['value'][-1],latitude=site_metadata['latitude']['value'][-1])
        else: 
            pytz_str, site_metadata['station_timezone']['value'][-1] = local_offset_from_utc(dt_ref,longitude=site_metadata['longitude']['value'][-1],latitude=site_metadata['latitude']['value'][-1])
        
    #save data_download_stamp --> rounded to nearest minute
    data_download_datetime = datetime.datetime.strptime(time.ctime(os.path.getmtime(data_file)), "%a %b %d %H:%M:%S %Y")
    data_download_datetime = round_to_minute(data_download_datetime)
    site_metadata['data_download_stamp']['value'][-1] = datetime_to_fractional_minutes_from_reference(data_download_datetime)
    
    return site_metadata
    
###------------------------------------------------------------------------------------###

#Module to check if any times are duplicate or overlapping across the time record.
#Start times are first sorted in ascending order to resolve potential overlapping times.
#Firstly, if a temporal window is identically duplicated (i.e. same start and end time), the first temporal window is kept preferentially .
#Secondly it is checked if any window end times overlap with the next start time. If an overlap is found, the finest temporal resolution period is kept preferentially. If more than 1 window of the same 'finest' resolution, then the first window is preferentially kept.
#These checks are done recursively until each window has no other windows which are duplicates or overlap with it. 
#Module also deals with re-ordering/modification of all variables which are time variant  

def process_overlapping_and_duplicate_times(data):

    #1. sort measurement start time (and associated time variant variables) in ascending order to resolve potential overlapping times
    
    #get data parameter names
    data_keys = data.dtype.names
    
    #get indices of sorted time array
    sorted_time_indices = np.array(sorted(range(len(data['time'])), key=lambda k: data['time'][k]))
    
    #sort time array and other data arrays
    data = data[sorted_time_indices]
    
    #-----------------------
    #2. Add temporal resolutions to start times to get measurement end times
    #check for measurement windows which are duplicated or overlap.
    #if have duplicate time windows, keep first value preferentially (write out qa flag, as it is unknown why there are duplicate times reported.
    #if there are any overlapping windows, this is because of different temporal resolutions. Preferentially keep finest resolution data.

    #iterate through time
    inds_to_remove = np.empty(0, dtype=np.uint32)
    for ii, start_time in enumerate(data['time'][:-1]):
        
        #only test for current time if not in list of bad indices to be removed
        if ii not in inds_to_remove:
        
            duplicate_inds = np.empty(0, dtype=np.uint32)
            overlap_inds = np.empty(0, dtype=np.uint32)
    
            #get window end time by adding temporal resolution as timedelta
            current_measurement_resolution = data['temporal_resolution'][ii]
            if current_measurement_resolution == 'H':
                timedelta = 60
            elif current_measurement_resolution == 'D':
                timedelta = 60*24
            elif current_measurement_resolution == 'W':
                timedelta = (60*24)*7
            elif current_measurement_resolution == 'M':
                #for monthly timedelta, add relative monthly timedelta from start time, then get difference in minutes between start and end time 
                dt_start = fractional_minutes_from_reference_to_datetime(float(start_time))
                dt_end = dt_start + relativedelta(months=+1)
                timedelta = ((dt_end - dt_start).days)*24*60
            else:
                timedelta = int(current_measurement_resolution)
            #get measurement window end time 
            end_time = start_time + timedelta 
    
            #1. check if any measurement window is duplicate, 
            #2. check if window end time overlaps with next start time,
            #keep checking until there are no overlaps/ duplicates with current window
            for jj, next_start_time in enumerate(data['time'][ii+1:]):
            
                #only test for next time if not in list of bad indices to be removed
                if ii+(jj+1) not in inds_to_remove:
            
                    #get window end time by adding temporal resolution as timedelta
                    next_measurement_resolution = data['temporal_resolution'][ii+(jj+1)]
                    if next_measurement_resolution == 'H':
                        next_timedelta = 60
                    elif next_measurement_resolution == 'D':
                        next_timedelta = 60*24
                    elif next_measurement_resolution == 'W':
                        next_timedelta = (60*24)*7
                    elif next_measurement_resolution == 'M':
                        #for monthly timedelta, add relative monthly timedelta from start time, then get difference in minutes between start and end time 
                        next_dt_start = fractional_minutes_from_reference_to_datetime(float(next_start_time))
                        next_dt_end = next_dt_start + relativedelta(months=+1)
                        next_timedelta = ((dt_end - dt_start).days)*24*60
                    else:
                        next_timedelta = int(next_measurement_resolution)
                    #get measurement window end time 
                    next_end_time = next_start_time + next_timedelta 
            
                    #duplicate check
                    if (start_time == next_start_time) & (end_time == next_end_time):
                        duplicate_inds=np.append(duplicate_inds,ii+(jj+1))    
                    #overlap check
                    elif end_time > next_start_time:
                        overlap_inds=np.append(overlap_inds,ii+(jj+1))
                    #if no duplicates or overlaps, proceed to next start time
                    else:
                        break
        
            #if have duplicate measurements keep first value preferentially (write out qa flag, as it is unknown why there are duplicate times reported. 
            if len(duplicate_inds) > 0:
                #write out qa flag for duplicate time that is preferentially kept
                data['qa'][ii] = data['qa'][ii]+standardisation_modules.get_standard_qa_flag('Duplicate Time - First Value Kept')
            
                #delete duplicate_inds from overlap_inds
                overlap_inds = [i for i in overlap_inds if i not in duplicate_inds]
            
                #save out indices to remove
                inds_to_remove = np.append(inds_to_remove, duplicate_inds)
        
            #if have overlapping measurement windows, first keep data preferentially by finest temporal resolution. if no difference in temporal resolutions,  then keep first time preferentially
            if len(overlap_inds) > 0:
                overlap_inds = np.append(ii,overlap_inds)
            
                #get temporal resolutions of overlapping time periods
                overlapping_window_resolutions = data['temporal_resolution'][overlap_inds]
            
                #first check if all the overlapping_window_resolutions are the same
                #if temporal resolutions are all the same, preferentially keep the first time
                if list(overlapping_window_resolutions).count(overlapping_window_resolutions[0]) == len(overlapping_window_resolutions):
                    #write out qa flag for overlapping time that is preferentially kept
                    data['qa'][ii] = data['qa'][ii]+standardisation_modules.get_standard_qa_flag('Overlapping Time - First Value Kept')
                    
                    #save out indices to remove
                    inds_to_remove = np.append(inds_to_remove, overlap_inds[1:])
                    
                #otherwise, if have different temporal resolutions, preferentially keep finest resolution data.
                #if have multiple times of the same fine resolution, the first time of these is preferentially kept
                else:
                    #convert all resolutions to integers for comparison
                    for rr, resolution in enumerate(overlapping_window_resolutions):
                        if resolution == 'H':
                            overlapping_window_resolutions[rr] = 60
                        elif resolution == 'D':
                            overlapping_window_resolutions[rr] = 60*24
                        elif resolution == 'W':
                            overlapping_window_resolutions[rr] = (60*24)*7
                        elif resolution == 'M':
                            dt_start = fractional_minutes_from_reference_to_datetime(float(data['time'][overlap_inds[rr]]))
                            dt_end = dt_start + relativedelta(months=+1)
                            overlapping_window_resolutions[rr] = ((dt_end - dt_start).days)*24*60
                        else:
                            overlapping_window_resolutions[rr] = int(overlapping_window_resolutions[rr])    
                    
                    #get index of measurement value to keep which has the finest temporal resolution - if have multiple times of the same fine resolution, the first time is preferentially kept
                    index_to_keep  = np.argmin(overlapping_window_resolutions)
                    
                    #write out qa flag for overlapping time that is preferentially kept
                    data['qa'][index_to_keep] = data['qa'][index_to_keep]+standardisation_modules.get_standard_qa_flag('Overlapping Time - Finest Temporal Resolution Kept')
                    
                    #generate list of the indices to delete - omitting the index to keep  
                    bad_inds = np.delete(overlap_inds,index_to_keep)            
            
                    #save out indices to remove
                    inds_to_remove = np.append(inds_to_remove, bad_inds)
            
    #if have indices to remove (either duplicates/or overlaps), do so
    if len(inds_to_remove) > 0:
        data = np.delete(data, inds_to_remove) 

    return data
  
###------------------------------------------------------------------------------------###

#module that processes data dictionary (i.e. metadata/data) to numpy dataframe
#this process also standardises the data formats of the raw data
#also does string formatting (i.e. upper, lower case strings).
#all missing elements are conserved as np.NaN values throughout processing, regardless of type

def convert_to_numpy_dataframe(data_dict):

    #create lists for standardising data formats of the input data dictionary
    standard_dtype = [(key,data_dict[key]['data_type']) for key in data_dict]

    #create numpy dataframe to store standardised forms of variables inside data dictionary
    standardised_dataframe = np.empty(len(data_dict[data_dict.keys()[0]]['value']), dtype=standard_dtype)
   
    #fill numpy dataframe with data
    for key in standardised_dataframe.dtype.names:
        
        #cut data for data_key
        param_array = data_dict[key]['value']
            
        #do string formatting for relevant fields before filling dataframe
        if standardised_dataframe[key].dtype == np.dtype('O'):
        
            #upper case string formatting
            if 'upper' in data_dict[key]['string_format']:
                param_array = [np.str(x).replace(",", "-").upper() if pd.isnull(x) == False else x for x in param_array]
        
            #lower case string formatting
            elif 'lower' in data_dict[key]['string_format']:
                param_array = [np.str(x).replace(",", "-").lower() if pd.isnull(x) == False else x for x in param_array]

            #titalling string formatting
            elif 'title' in data_dict[key]['string_format']:  
                param_array = [string.capwords(np.str(x)).replace(",", "-") if pd.isnull(x) == False else x for x in param_array]
        
        #fill dataframe
        standardised_dataframe[key] = param_array
        
    return standardised_dataframe
        
###------------------------------------------------------------------------------------###

#write out data to temporary/final station specific numpy compressed .npz file
    
def write_station_specific_numpy(metadata,data):
    
    #get station reference
    station_reference = metadata['station_reference'][0]
        
    #write numpy file
    if write_stage == 'first_write':
    
        print '%s -- WRITING TO PRIMARY .npz'%(station_reference)
        
        temporary_file_exists = os.path.isfile('/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference))
        
        #deal with instances where temporary station file already exists
        if temporary_file_exists == True:
            #when processing files with all sites per year), then append to existing file
            if network_details['file_storage_type'] == 'all_per_year':
                    #read existing file
                    existing_metadata = read_from_numpy('/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference), 'metadata')
                    existing_data =     read_from_numpy('/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference), 'data')
                    #remove existing file
                    os.remove('/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference))
                    #append current metadata/data to existing metadata/data
                    existing_metadata_n_rows = existing_metadata.shape[0]
                    existing_data_n_rows = existing_data.shape[0]
                    metadata_n_rows = metadata.shape[0]
                    data_n_rows = data.shape[0]
                    #resize existing arrays to get ready for concatenating new arrays
                    existing_metadata.resize(existing_metadata_n_rows+metadata_n_rows)
                    existing_data.resize(existing_data_n_rows+data_n_rows)
                    existing_metadata[-metadata_n_rows:] = metadata
                    existing_data[-data_n_rows:] = data
                    #write out
                    np.savez_compressed('/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference),data=existing_data,metadata=existing_metadata)
        
            #when processing files with one site per year, and file already exists then write new data/metadata to separate file (with appended string)
            elif (network_details['file_storage_type'] == 'site_per_year') or (network_details['file_storage_type'] == 'site_per_multiyear'):   
                
                print 'PRIMARY .npz ALREADY EXISTS FOR %s, WRITING TO SEPARATE FILE'%(station_reference)
                
                #determine if other duplicate files exist
                dup_files = glob.glob('/esarchive/obs/ghost/%s/native/%s/temporary/%s*'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference))
                n_dup = len(dup_files)
                
                #write new file with '_dupN' string appended to filename
                np.savez_compressed('/esarchive/obs/ghost/%s/native/%s/temporary/%s_dup%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference,n_dup),data=data,metadata=metadata)
        
                
        #if temporary station file does not yet exist, write file for the first time
        elif temporary_file_exists == False:
            np.savez_compressed('/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference),data=data,metadata=metadata)
        

    if write_stage == 'final_write':
    
        print '%s -- WRITING TO FINAL .npz'%(station_reference)
    
        #remove some data variables from dataframe that no longer need (as have been used when creating QA flags)
        data = rf.drop_fields(data, ['reported_lower_limit_of_detection_per_measurement','reported_upper_limit_of_detection_per_measurement'])
        
        #write out
        np.savez_compressed('/esarchive/obs/ghost/%s/native/%s/%s_%s_%s_%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference,parameter_details['bsc_parameter_name'],run_details['start_year'],run_details['end_year']),data=data,metadata=metadata)

    return

###------------------------------------------------------------------------------------###

#standardise metadata/data, check for overlapping/duplicate times, and write out site data to temporary numpy file

def post_process_station_data(metadata,data,station_reference):
    
    #standardise data formats - convert to numpy dataframes
    metadata = convert_to_numpy_dataframe(metadata)
    data = convert_to_numpy_dataframe(data) 

    #check for duplicated/overlapping times across whole time record
    data = process_overlapping_and_duplicate_times(data)    
    
    #write out to temporary station specific numpy file
    write_station_specific_numpy(metadata,data)
    
    #if are running in parallel - write out station reference to parallel_safe_dir to inform station has been processed
    #if 'PARALLEL' in run_details['process_type']:
    #    stem = '/esarchive/obs/ghost/%s/native/%s/parallel_safe/'%(run_details['network'],parameter_details['bsc_parameter_name'])
    #    f= open("%s%s"%(stem,station_reference),"w+")
    #    f.close() 
    
    return
    
###------------------------------------------------------------------------------------###

#resolve issues where have multiple temporary numpy files for a station 
#this happens when station codes are not known before reading the data files 
#if none of files overlap in time, concatenate data together.

def resolve_duplicate_reference_files(duplicate_reference):

    print 'DEALING WITH MULTIPLE PRIMARY .npz FILES for %s'%(duplicate_reference)
    
    #get all duplicate reference files
    duplicate_files = sorted(glob.glob('/esarchive/obs/ghost/%s/native/%s/temporary/%s*'%(run_details['network'],parameter_details['bsc_parameter_name'], duplicate_reference)))
    all_start_times = []
    all_data_lengths = []
    overlapping_data = False
    #load each duplicate reference file and evaluate times (should not be overlapping!)
    for file_ii, duplicate_file in enumerate(duplicate_files):
        file_data = read_from_numpy(duplicate_file, 'data')
        all_data_lengths.append(len(file_data['time']))
        start_time = file_data['time'][0]
        end_time = file_data['time'][-1]
        all_start_times.append(start_time)
        if file_ii == 0:
            min_time = start_time
            max_time = end_time
        else:
            if end_time < min_time:
                min_time = start_time
            elif start_time > max_time:
                max_time = end_time
            else:
                overlapping_data = True
                if start_time < min_time:
                    overlap_start = min_time 
                    min_time = start_time
                elif start_time >= min_time:
                    overlap_start = start_time
                    
                if end_time > max_time:
                    overlap_end = max_time
                    max_time = end_time
                elif end_time <= max_time:
                    overlap_end = end_time
                
    #if have not detected overlapping times, therefore concatenate data (in order of processed start times processed beforehand)
    if overlapping_data == False:
        print 'CONCATENATING FILE DATA FOR %s'%(duplicate_reference)
        
        duplicate_files = [x for _,x in sorted(zip(all_start_times,duplicate_files))]
        for file_ii, duplicate_file in enumerate(duplicate_files):
            if file_ii == 0:
                existing_data = read_from_numpy(duplicate_file, 'data')
                existing_metadata = read_from_numpy(duplicate_file, 'metadata')
                #remove read file
                os.remove(duplicate_file)
            else:
                data = read_from_numpy(duplicate_file, 'data')
                metadata = read_from_numpy(duplicate_file, 'metadata')
                #remove read file
                os.remove(duplicate_file)
                #append current metadata/data to existing metadata/data
                existing_metadata_n_rows = existing_metadata.shape[0]
                existing_data_n_rows = existing_data.shape[0]
                metadata_n_rows = metadata.shape[0]
                data_n_rows = data.shape[0]
                #resize existing arrays to get ready for concatenating new arrays
                existing_metadata.resize(existing_metadata_n_rows+metadata_n_rows)
                existing_data.resize(existing_data_n_rows+data_n_rows)
                existing_metadata[-metadata_n_rows:] = metadata
                existing_data[-data_n_rows:] = data
                
        #write out
        np.savez_compressed('/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],duplicate_reference),data=existing_data,metadata=existing_metadata)

    #if have overlapping times, keep files with longest series of time series data preferentially
    elif overlapping_data == True:
        file_to_keep = duplicate_files[np.argmax(all_data_lengths)]
        print 'OVERLAPPING FILE DATA FOR %s between %s and %s. KEEPING FILE WITH LONGEST CONTINUOUS TIME SERIES = %s '%(duplicate_reference, fractional_minutes_from_reference_to_datetime(float(overlap_start)), fractional_minutes_from_reference_to_datetime(float(overlap_end)), file_to_keep)
        files_to_not_keep = duplicate_files[:np.argmax(all_data_lengths)]+duplicate_files[np.argmax(all_data_lengths)+1:]
        #remove files that you are not keeping
        for bad_file in files_to_not_keep:
            os.remove(bad_file)
        #change filename of file you are keeping to remove "_dup" 
        desired_filename = '/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],duplicate_reference)
        os.rename(file_to_keep, desired_filename)
            
    return 

###------------------------------------------------------------------------------------###
#read numpy compressed .npz files into memory.
#return either data or metadata as desired

def read_from_numpy(numpy_file, data_type):

    numpy_dataframe = np.load(numpy_file)
    if data_type == 'metadata':
        data_array = numpy_dataframe['metadata']
    elif data_type == 'data':
        data_array = numpy_dataframe['data']
    
    return data_array
    
###------------------------------------------------------------------------------------###

#module processes reported uncertainty per measurement from metadata reported (or if not available, documented) uncertainties, if not already given per measurement by network . 

#module also derives +/- uncertainty on each measurement from the reported (or if not available, documented) accuracy and precision metadata.
#precision and accuracy are added in quadrature to generate uncertainties per measurement.
#calculation is only done if have both valid precision and accuracy information
 
def process_reported_and_derived_uncertainty_per_measurement(metadata,data):

    #get key data
    reported_uncertainty_per_measurement = data['reported_uncertainty_per_measurement']

    #get key metadata 
    reported_uncertainty = metadata['measuring_instrument_reported_uncertainty']
    documented_uncertainty = metadata['measuring_instrument_documented_uncertainty']
    reported_precision = metadata['measuring_instrument_reported_precision']
    documented_precision = metadata['measuring_instrument_documented_precision']
    reported_accuracy = metadata['measuring_instrument_reported_accuracy']
    documented_accuracy = metadata['measuring_instrument_documented_accuracy']
    update_metadata_stamp = metadata['meta_update_stamp']
    
    #------------------------------------------------------------#
    #process reported uncertainties
    
    #iterate through each metadata parameter
    for ii in range(len(reported_uncertainty)):
    
        #do you have valid reported metadata uncertainty?
        if pd.isnull(reported_uncertainty[ii]) == False:
            uncertainty = reported_uncertainty[ii]
        #do you have valid documented metadata uncertainty?
        elif pd.isnull(documented_uncertainty[ii]) == False:
            uncertainty = documented_uncertainty[ii]
        #no valid uncertainty value, continue to next metadata index
        else:
            continue
    
        #get relevant range of inds to update reported_uncertainty_per_measurement data field
        #is it last index? 
        if ii == (len(reported_precision)-1):
            relevant_inds = np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), len(data['time']))
        else:
            relevant_inds = np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), bisect.bisect_left(data['time'], update_metadata_stamp[ii+1]))
    
        #if uncertainty is in percentage terms, convert to absolute terms (for all values across relevant range)
        #determine type of way uncertainty is reported, and deal with processing accordingly
        
        #get all string characters (no numbers) 
        uncertainty_char = ''.join([i for i in uncertainty if not i.isdigit()]).replace('.','')
    
        #put all separate numerical values in string as elements in new list
        uncertainty_val = np.array(re.findall(r"[+-]?\d+(?:\.\d+)?", uncertainty)).astype(np.float32)
    
        #1. absolute value
        if uncertainty_char == '':
            uncertainty_absolute = uncertainty_val[0] * np.ones(len(relevant_inds))
    
        #2. percentage value
        elif uncertainty_char == '%':
            fraction = uncertainty_val[0]/100.0
            uncertainty_absolute = data[parameter_details['bsc_parameter_name']][relevant_inds]*fraction
        
        #3. the greater of either an absolute value or percentage
        elif (uncertainty_char == '/%') or (uncertainty_char == '%/'):
            
            if quantity_char == '/%':
                absolute = uncertainty_val[0]
                fraction = uncertainty_val[1]/100.0
                
            elif quantity_char == '%/':
                absolute = uncertainty_val[1]
                fraction = uncertainty_val[0]/100.0  
                
            uncertainty_absolute = []
            for val in data[parameter_details['bsc_parameter_name']][relevant_inds]:
                #percentage (as absolute) greater
                if (fraction*val) > absolute:
                    uncertainty_absolute=np.append(uncertainty_absolute,fraction*val) 
                #absolute greater
                else:
                    uncertainty_absolute=np.append(uncertainty_absolute,absolute)
                
        #4. percentage value after a fixed limit (minimum uncertainty)
        elif uncertainty_char == '%>=':
            limit = uncertainty_val[1]
            fraction = uncertainty_val[0]/100.0
            
            uncertainty_absolute = []
            for val in data[parameter_details['bsc_parameter_name']][relevant_inds]:
                #percentage (as absolute) higher than limit 
                if (fraction*val) > limit:
                    uncertainty_absolute=np.append(uncertainty_absolute,fraction*val)
                #not greater than limit
                else:
                    uncertainty_absolute=np.append(uncertainty_absolute,limit)
                    
        #5. choice of 2 absolute values under/over fixed limit
        elif uncertainty_char_char == '< >=':
            absolute_under =  uncertainty_val[0]
            limit =     uncertainty_val[1]
            absolute_over =  uncertainty_val[2]
            
            uncertainty_absolute = []
            for val in data[parameter_details['bsc_parameter_name']][relevant_inds]:
                #absolute value higher than limit 
                if val >= limit:
                    uncertainty_absolute=np.append(uncertainty_absolute,absolute_over)
                #absolute value under limit
                elif val < limit:
                    uncertainty_absolute=np.append(uncertainty_absolute,absolute_under)
                    
        #overwrite 'reported_uncertainty_per_measurement' field if == NaN
        for rr, relevant_ind in enumerate(relevant_inds):
            if pd.isnull(data['reported_uncertainty_per_measurement'][relevant_ind]) == True:
                data['reported_uncertainty_per_measurement'][relevant_ind] = uncertainty_absolute[rr]

    #------------------------------------------------------------#
    #process derived uncertainties
    
    #iterate through each metadata parameter
    for ii in range(len(reported_precision)):
    
        #do you have valid reported precision?
        if pd.isnull(reported_precision[ii]) == False:
            precision = reported_precision[ii]
        #do you have valid documented precision?
        elif pd.isnull(documented_precision[ii]) == False:
            precision = documented_precision[ii]
        #no valid precision value, continue to next metadata index
        else:
            continue
            
        #do you have valid reported accuracy?
        if pd.isnull(reported_accuracy[ii]) == False:
            accuracy = reported_accuracy[ii]
        #do you have valid documented accuracy?
        elif pd.isnull(documented_accuracy[ii]) == False:
            accuracy = documented_accuracy[ii]
        #no valid accuracy value, continue to next metadata index
        else:    
            continue
        
        #get relevant range of inds to update derived_uncertainty_per_measurement data field
        #is it last index? 
        if ii == (len(reported_precision)-1):
            relevant_inds = np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), len(data['time']))
        else:
            relevant_inds = np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), bisect.bisect_left(data['time'], update_metadata_stamp[ii+1]))
    
    
        #if precision or accuracy are in percentage terms, convert to absolute terms (for all values across relevant range)
        #determine type of way precision/accuracy are reported, and deal with processing accordingly
        
        quantities = [precision,accuracy]
        quantities_absolute = []
        
        for quantity in quantities:
            
            #get all string characters (no numbers) 
            quantity_char = ''.join([i for i in quantity if not i.isdigit()]).replace('.','')
        
            #put all separate numerical values in string as elements in new list
            quantity_val = np.array(re.findall(r"[+-]?\d+(?:\.\d+)?", quantity)).astype(np.float32)
        
            #1. absolute value
            if quantity_char == '':
                quantity_absolute = quantity_val[0] * np.ones(len(relevant_inds))
        
            #2. percentage value
            elif quantity_char == '%':
                fraction = quantity_val[0]/100.0
                quantity_absolute = data[parameter_details['bsc_parameter_name']][relevant_inds]*fraction
            
            #3. the greater of either an absolute value or percentage
            elif (quantity_char == '/%') or (quantity_char == '%/'):
                
                if quantity_char == '/%':
                    absolute = quantity_val[0]
                    fraction = quantity_val[1]/100.0
                    
                elif quantity_char == '%/':
                    absolute = quantity_val[1]
                    fraction = quantity_val[0]/100.0  
                    
                quantity_absolute = []
                for val in data[parameter_details['bsc_parameter_name']][relevant_inds]:
                    #percentage (as absolute) greater
                    if (fraction*val) > absolute:
                        quantity_absolute=np.append(quantity_absolute,fraction*val) 
                    #absolute greater
                    else:
                        quantity_absolute=np.append(quantity_absolute,absolute)
                    
            #4. percentage value after a fixed limit (minimum uncertainty)
            elif quantity_char == '%>=':
                limit = quantity_val[1]
                fraction = quantity_val[0]/100.0
                
                quantity_absolute = []
                for val in data[parameter_details['bsc_parameter_name']][relevant_inds]:
                    #percentage (as absolute) higher than limit 
                    if (fraction*val) >= limit:
                        quantity_absolute=np.append(quantity_absolute,fraction*val)
                    #not greater than limit
                    else:
                        quantity_absolute=np.append(quantity_absolute,limit)
                        
            #5. choice of 2 absolute values under/over fixed limit
            elif quantity_char == '< >=':
                absolute_under =  quantity_val[0]
                limit =     quantity_val[1]
                absolute_over =  quantity_val[2]
                
                quantity_absolute = []
                for val in data[parameter_details['bsc_parameter_name']][relevant_inds]:
                    #absolute value higher than limit 
                    if val >= limit:
                        quantity_absolute=np.append(quantity_absolute,absolute_over)
                    #absolute value under limit
                    elif val < limit:
                        quantity_absolute=np.append(quantity_absolute,absolute_under)
        
            quantities_absolute.append(quantity_absolute)


        #calculate derived uncertainty -- sqrt((precision**2) + (accuracy**2))
        uncertainty = np.sqrt((quantities_absolute[0]**2)+(quantities_absolute[1]**2))
    
        #update field
        data['derived_uncertainty_per_measurement'][relevant_inds] = uncertainty

    return data
    
###------------------------------------------------------------------------------------###

#check if key metadata (Latitude, Longitude, Measurement Altitude (Altitude + Sampling Height) changes significantly through time record.

#significant change in Latitude/Longitude coordinates is defined to be >= 0.0001 degrees (~11m)
#significant change in Measurement Altitude is defined to be >= 11m.

#if all of one of the Latitude, Longitude, Altitude columns are NaNs, do not run module. 
#QA testing will subsequently remove station from processing.

#if one of the Latitude, Longitude or Altitude values is NaN temporally modify them to be the most recent valid value, or if not possible the next valid value. 
#this stops stations being split out due to missing fields which may have erroneously being left out (or metadata reporting has improved with time)
#it is therefore assumed that the most recent valid metadata value, or next valid value is applicable for the missing period.

#Measurement Altitude is station altitude + measurement height (where available).
#by default measurement altitude is simply altitude, if sampling heights are not available.
#if altitude is not available, but sampling heights are, then measurement altitude is deemed NaN.

#check that Latitude, Longitude, Measurement Altitude does not change then revert back at later date. 
#therefore group different collections of Latitudes, Longitudes & Measurement Altitudes together - output as 1 unique station 
#this prevents writing of multiple new stations needlessly.

#finally split out data/metadata into new stations if required.

def changing_key_metadata_new_stations(metadata,data):
    
    #station_reference
    station_reference = metadata['station_reference'][0]
    
    #number of stations (may be modified later in code) 
    n_sites = 1
    
    #tolerance limit for changing significant lat/lon coordinate changes 
    latlon_tolerance = 0.0001 #~11m
    #tolerance limit for changing significant measurement altitude changes 
    measurement_alt_tolerance = 11.0 #11m
    
    #arrays for appending new_metadata/data 
    new_metadata = []
    new_data = []
        
    #array for saving indices for deleting original metadata/data arrays
    del_indices = []
    
    #get key metadata 
    lats = metadata['latitude']
    lons = metadata['longitude'] 
    alts = metadata['altitude'] 
    sampling_heights = metadata['sampling_height']
    key_meta = [lats,lons,alts,sampling_heights]
     
    #if all of one of the key metadata fields (latitude, longitude, altitude) are all Nan, then do not proceed further with function
    #QA testing will subsequently remove station from processing 
    if (all(pd.isnull(lats) == True)) or (all(pd.isnull(lons) == True)) or (all(pd.isnull(alts) == True)):
        return metadata, data, n_sites
        
    #if any of the lat/lon/altitude/sampling heights are NaN values, modify them to be the most recent valid value, or if not possible the next valid value 
    #this stops stations being split out due to missing fields which may have erroneously being left out (or metadata reporting has improved with time)
    #it is therefore assumed that the most recent valid metadata value, or next valid value is applicable for the missing period.
    for ii,key_meta_field in enumerate(key_meta):
        for jj,val in enumerate(key_meta_field):
            #if value is NaN, first check for most recent valid value (going backwards in time)
            if pd.isnull(val) == True:
                for check_valid_var_back in key_meta[ii][jj-1::-1]:
                    #if have found valid value overwrite it temporally to the current time window for purposes of this module
                    if pd.isnull(check_valid_var_back) == False:
                        key_meta[ii][jj] = check_valid_var_back
                        break
                #if have not found valid value going backwards in time, check forwards in time
                for check_valid_var_forward in key_meta[ii][jj+1:]:
                #if have found valid value overwrite it temporally to the current time window for purposes of this module  
                    if pd.isnull(check_valid_var_forward) == False:
                        key_meta[ii][jj] = check_valid_var_forward
    
    #if have valid sampling height and altitude, create a measurement altitude
    #by default measurement altitude is simply altitude, if sampling heights are not available.
    measurement_alts = copy.deepcopy(alts)
    for ii in range(len(measurement_alts)):
        if (pd.isnull(sampling_heights[ii]) == False) & (pd.isnull(alts[ii]) == False):
            measurement_alts[ii] = sampling_heights[ii] + alts[ii]
   
    #arrays for grouping together indices that are within tolerance limits
    lat_group_inds = []
    lon_group_inds = []
    measurement_alt_group_inds = []
    
    #iterate through each lat/lon/measurement altitude
    for ii in range(len(lats)):
        
        reference_lat, reference_lon, reference_measurement_alt = lats[ii], lons[ii], measurement_alts[ii]
    
        #add current index to reference group inds
        lat_group_inds.append([ii])
        lon_group_inds.append([ii])
        measurement_alt_group_inds.append([ii])
        
        other_inds = np.delete(range(len(lats)),ii)
        
        #iterate through all other lat/lon/measurement altitudes
        #is each lat/lon/measurement height within tolerance limit of current reference lat/lon/measurement altitude?
        #if so then add current index to reference group inds
        for jj in other_inds:
            
            #get differences
            lat_diff = np.abs(reference_lat - lats[jj])
            lon_diff = np.abs(reference_lon - lons[jj])
            measurement_alt_diff = np.abs(reference_measurement_alt - measurement_alts[jj])
            
            #if differences are smaller than tolerance, add current index to group inds
            if (lat_diff < latlon_tolerance) & (pd.isnull(lat_diff) == False):
                lat_group_inds[-1].append(jj)
                
            if (lon_diff < latlon_tolerance) & (pd.isnull(lon_diff) == False):
                lon_group_inds[-1].append(jj)
                
            if (measurement_alt_diff < measurement_alt_tolerance) & (pd.isnull(measurement_alt_diff) == False):
                measurement_alt_group_inds[-1].append(jj)
    

    #find intersect indices for latitude, longitude and measurement altitudes, iteratively
    intersect_inds = []
    for xx in range(len(lat_group_inds)):
        intersect_inds.append(list(set(lat_group_inds[xx]) & set(lon_group_inds[xx]) & set(measurement_alt_group_inds[xx])))
        
    #are any of the lists duplicates? if so, remove duplicates (prevents needless creation of new stations if metadata changes then reverts back)
    intersect_inds.sort()
    intersect_inds = list(k for k,_ in itertools.groupby(intersect_inds))
    
    #if more than 1 list, then split out metadata/data into different stations
    #after sort, first list of indices always contains the zero index (i.e. the first reported metadata), so this remains as the original station
    if len(intersect_inds) > 1:
    
        n_sites = len(intersect_inds)
    
        #iterate through new station metadata indices lists
        for inter_i in range(len(intersect_inds)):
        
            #frames for containing split out metadata/data
            new_metadata_frame = [[] for _ in xrange(len(metadata_keys))]
            new_data_frame =     [[] for _ in xrange(len(data_keys))]
            
            new_metadata_inds = intersect_inds[inter_i]
            
            #get indices to split metadata/data arrays
            #for data - do so by finding where reference times line up with metadata_update_timestamps 
            #for metadata - simply lowest index
            start_data_ind = bisect.bisect_left(data['time'] == metadata['meta_update_stamp'][np.min(new_metadata_inds)])
            start_metadata_ind = np.min(new_metadata_inds)
            
            #what is highest index in list?
            max_new_metadata_inds = np.max(new_metadata_inds)
            #is it the last metadata update timestamp? if so then iterate to end of metadata/data arrays
            if max_new_metadata_inds == (len(lats)-1):
                end_data_ind = len(data['time'])
                end_metadata_ind = len(lats)
            #otherwise get index of end metadata/data index using next index
            else:
                end_data_ind = bisect.bisect_left(data['time'] == metadata['meta_update_stamp'][max_new_metadata_inds+1])
                end_metadata_ind = max_new_metadata_inds+1
        
            #cut new metadata/data dataframe from original numpy dataframe
            new_metadata_frame = metadata[start_metadata_ind:end_metadata_ind]
            new_data_frame = data[start_data_ind:end_data_ind]
            
            #change station reference metadata (if not first station in time)
            if inter_i > 0:
                for qq in range(len(new_metadata_frame['station_reference'])):
                    new_metadata_frame['station_reference'][qq] = '%sN%s'%(station_reference,inter_i)
            
            #append to list containing multiple numpy dataframes for separated station instances
            new_metadata.append(new_metadata_frame)
            new_data.append(new_data_frame)
        
        #return new metadata/data arrays
        return new_metadata, new_data, n_sites
    
    #if no metadata changes significantly, return original metadata/data
    else:
        return metadata, data, n_sites
    
    
###------------------------------------------------------------------------------------###
#read temporary specific station numpy .npz file, do processing and save out to finalised station specific numpy .npz file(s) 
#processing involves quality control checks and checking if need to separate out stations due to changing latitude/longitude/altitude metadata

def quality_control_processing(station_reference):
    
    #read station data from station 
    station_numpy_filename = '/esarchive/obs/ghost/%s/native/%s/temporary/%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference)
    
    #take numpy file and separate into numpy record arrays for metadata/data
    metadata = read_from_numpy(station_numpy_filename, 'metadata')
    data =     read_from_numpy(station_numpy_filename, 'data')
    
    #check for duplicated/overlapping times across whole time record
    data = process_overlapping_and_duplicate_times(data) 
    
    #process reported and derived uncertainty per measurement
    data = process_reported_and_derived_uncertainty_per_measurement(metadata,data)
    
    #process stations with significantly changing lat/lon/alt to new sites
    metadata, data, n_sites = changing_key_metadata_new_stations(metadata,data)
    
    #delete all current processed final numpy files with same initial reference code
    #this is done to prevent accumulating files for stations for which the data are split out, this is possible when changing threshold limits for the splitting process
    for zippath in glob.iglob(os.path.join('/esarchive/obs/ghost/%s/native/%s/'%(run_details['network'],parameter_details['bsc_parameter_name']), '%s*'%(station_reference))):
        os.remove(zippath)

    #array for saving calculated modal metadata
    #all_modal_metadata = []

    #iterate through number of split out stations
    for i in range(n_sites):
    
        if n_sites > 1:
            metadata_cut, data_cut = metadata[i], data[i]
        else:
            metadata_cut, data_cut = metadata, data
        
        print '%s -- QC'%(metadata_cut['station_reference'][0])
        
        #process site data through quality control checks
        data_cut, exit_flag = quality_control_modules.do_qa(metadata_cut,data_cut)

        if exit_flag == False:
            
            #write out to station specific numpy file
            write_station_specific_numpy(metadata_cut,data_cut)
            
            #calculate modal metadata
            #modal_metadata = calculate_static_modal_metadata(metadata_cut,data_cut)
            #if len(all_modal_metadata) == 0:
            #    all_modal_metadata = modal_metadata
            #else:
            #    all_modal_metadata = np.vstack((all_modal_metadata, modal_metadata))
            
            #write out station reference to inform station has been written out to final numpy file (by split out reference name - if applicable)
            stem = '/esarchive/obs/ghost/%s/native/%s/final_process_valid/'%(run_details['network'],parameter_details['bsc_parameter_name'])
            f= open("%s%s"%(stem,metadata_cut['station_reference'][0]),"w+")
            f.close() 
            
    #delete temporary station numpy file - now that processing is finished
    os.remove(station_numpy_filename)
    
    return 
   
###------------------------------------------------------------------------------------###
#read finalised station specific .npz files, and process through classification checks, re-saving files with new classification information.  

def classification_processing(station_reference):

    print '%s -- CLASSIFICATIONS'%(station_reference)

    #read station data from station 
    station_numpy_filename = '/esarchive/obs/ghost/%s/native/%s/%s_%s_%s_%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference,parameter_details['bsc_parameter_name'],run_details['start_year'],run_details['end_year'])
    
    #take numpy file and separate into numpy record arrays for metadata/data
    metadata = read_from_numpy(station_numpy_filename, 'metadata')
    data =     read_from_numpy(station_numpy_filename, 'data')
    
    #process station data through classification checks
    data = classification_modules.do_classifications(metadata,data)
    
    #rewrite out to station specific numpy file
    write_station_specific_numpy(metadata,data)
    
    return
    
###------------------------------------------------------------------------------------###

def calculate_static_modal_metadata(metadata,data):

    #station_reference
    station_reference = metadata['station_reference'][0]

    #get metadata standards
    metadata_standards = standardisation_modules.create_standardised_metadata_frame()
    
    #get metadata update stamps
    update_metadata_stamp = metadata['meta_update_stamp']
    
    #get modal metadata column header keys
    metadata_provisional_keys = list(metadata_standards.keys())
    metadata_keys = []
    #limit metadata fields to just those that is wanted to be wrote out
    for key in metadata_provisional_keys:
        if pd.isnull(metadata_standards[key]['metadata_order']) == False:
            metadata_keys.append(key)
    
    #list for placing calculated modal metadata
    modal_metadata = [''] * len(metadata_keys)
    
    #calculate data_start_stamp and data_end_stamp
    data_start_stamp = data['time'][0]
    data_end_time = data['time'][-1] 
    data_end_res = data['temporal_resolution'][-1] 
    if data_end_res == 'H':
        data_end_stamp = data_end_time + 60
    elif data_end_res == 'D':
        data_end_stamp = data_end_time + (60*24)
    elif data_end_res == 'W':
        data_end_stamp = data_end_time + (60*24*7)
    elif data_end_res == 'M':
        dt_end_dt = fractional_minutes_from_reference_to_datetime(float(data_end_time))
        data_end_window_dt = dt_end_dt + relativedelta(months=+1)
        timedelta = ((data_end_window_dt - dt_end_dt).days)*24*60
        data_end_stamp = data_end_time + timedelta
    else:
        data_end_stamp = data_end_time + int(data_end_res)
    
    #iterate though metadata fields getting modal metadata
    for key in metadata_keys:
    
        #metadata values for field
        metadata_values = metadata[key]
        
        #take mode of weighted metadata (by number of minutes that it is relevant for) that does not equal 'nan'
        #if are multiple items with same number of instances, the lowest sorted instance will be returned (i.e. 'a' rather 'b'; 1 rather than 2)
        weighted_meta = {}
        for xx, metadata_value in enumerate(metadata_values):
            #last index
            if (xx+1) == len(metadata_values):
                minutes_represented = data_end_stamp - update_metadata_stamp[xx]
            else:
                minutes_represented = update_metadata_stamp[xx+1] - update_metadata_stamp[xx]
            
            if pd.isnull(metadata_value) == False:
                if metadata_value in weighted_meta.keys():
                    weighted_meta[metadata_value] = weighted_meta[metadata_value]+minutes_represented
                else:
                    weighted_meta[metadata_value] = minutes_represented
        
        #if have no valid metadata, modal value is np.NaN
        if len(weighted_meta) == 0:
            modal_value = np.NaN
        else:
            counter_instance = Counter(weighted_meta)
            modal_value = counter_instance.most_common(1)[0][0]
    
        #put calculated modal value into modal metadata list, placed in desired list position
        modal_metadata[metadata_standards[key]['metadata_order']] = modal_value
    
    #convert model metadata list to numpy object type
    modal_metadata = np.array(modal_metadata, dtype=np.object)
    
    return modal_metadata
    
###------------------------------------------------------------------------------------###

def write_static_modal_metadata(all_modal_metadata):

    print 'WRITING STATIC MODAL METADATA .csv'

    #get metadata standards
    metadata_standards = standardisation_modules.create_standardised_metadata_frame()

    #remove potentially empty lists in all_modal_metadata array
    all_modal_metadata = [x for x in all_modal_metadata if x != []]

    #sort modal metadata rows alphabetically by station_reference (first element)
    all_modal_metadata = sorted(all_modal_metadata, key=lambda x: x[0])
    
    #try reading existing csv file for network
    csv_filename = '/esarchive/obs/ghost/%s/metadata/processed/%s_modal_metadata.csv'%(run_details['network'],run_details['network'])
    try:
        network_modal_metadata = np.loadtxt(csv_filename,dtype=np.object,delimiter=",")
        sorted_column_names = network_modal_metadata[0,:]
        network_modal_metadata = network_modal_metadata[1:,:]
    
    #if file does not exist, create a new one with modal metadata from all current processed stations and return
    except:
        with open(csv_filename, "wb") as f:
            writer = csv.writer(f,delimiter=',',)
            
            #get modal metadata column header keys
            metadata_provisional_keys = list(metadata_standards.keys())
            metadata_keys = []
            #limit metadata fields to just those that is wanted to be wrote out
            for key in metadata_provisional_keys:
                if pd.isnull(metadata_standards[key]['metadata_order']) == False:
                    metadata_keys.append(key)

            #create array of sorted column header names
            sorted_column_names = [''] * len(metadata_keys)
            for key in metadata_keys:
                #add information to lists for reading existing modal metadata csv file
                sorted_column_names[metadata_standards[key]['metadata_order']] = key
            #change some column names to allow functionality with ET tool
            sorted_column_names[sorted_column_names.index('station_reference')] = 'station.code'  
            sorted_column_names[sorted_column_names.index('latitude')] = 'lat'    
            sorted_column_names[sorted_column_names.index('longitude')] = 'lon'   
            sorted_column_names[sorted_column_names.index('station_name')] = 'station.name'   
            
            #write column headers
            writer.writerow(sorted_column_names)
            #write station modal metadata to each line
            for modal_metadata in all_modal_metadata:
                writer.writerow(modal_metadata)
        return
    
    #if file previously existed, is there a line in file for each station reference ('station.code') processed? 
    rewrite_flag = False
    processed_station_references = network_modal_metadata[:,np.where(sorted_column_names=='station.code')[0][0]]
    #iterate through modal metadata for each station
    for modal_metadata in all_modal_metadata:
        station_reference = modal_metadata[0]
    
        #is current reference in processed references?
        if station_reference in processed_station_references:
    
            #if there is, check if metadata needs updating
            file_station_metadata = network_modal_metadata[np.where(processed_station_references == station_reference)[0][0],:]
            #test for equivalence
            if (file_station_metadata == modal_metadata).all() == False:
                #update data array with updated data for station
                network_modal_metadata[np.where(processed_station_references == station_reference)[0][0],:] = modal_metadata
        
                #set flag to rewrite file at end of process
                rewrite_flag = True
    
        #if there isn't a line for current station reference ('station.code'), create one    
        else:
        
            #stack file data with new station data
            network_modal_metadata = np.vstack((network_modal_metadata, modal_metadata))
                
            #set flag to rewrite file at end of process
            rewrite_flag = True
        
    #rewrite file if it needs updating
    if rewrite_flag == True:
        
        #re-sort lines alphabetically by station_reference (first element)
        network_modal_metadata = sorted(network_modal_metadata, key=lambda x: x[0])
        
        with open(csv_filename, "wb") as f:
            writer = csv.writer(f,delimiter=',',)
            #write column headers
            writer.writerow(sorted_column_names)
            #write line by line to csv
            for modal_metadata in network_modal_metadata:
                writer.writerow(modal_metadata)

    return 

###------------------------------------------------------------------------------------###

def process_modal_metadata(station_reference):

    print '%s -- CALCULATING MODAL METADATA'%(station_reference)

    #read station data from station 
    station_numpy_filename = '/esarchive/obs/ghost/%s/native/%s/%s_%s_%s_%s.npz'%(run_details['network'],parameter_details['bsc_parameter_name'],station_reference,parameter_details['bsc_parameter_name'],run_details['start_year'],run_details['end_year'])
    
    #take numpy file and separate into numpy record arrays for metadata/data
    metadata = read_from_numpy(station_numpy_filename, 'metadata')
    data =     read_from_numpy(station_numpy_filename, 'data')
    
    #calculate modal metadata for station
    modal_metadata = calculate_static_modal_metadata(metadata,data)

    return

###------------------------------------------------------------------------------------###

#module for converting native mixed temporal resolution data onto a grid of times of fixed intervals, between a start and end time in UTC time
#the temporal resolution of the grid can be either hourly (H), daily (D) or monthly (M)
#data with a resolution coarser than the temporal resolution desired is screened out.
#when converting from higher temporal resolution to lower temporal resolution, some processing is done:
#1. scientifically invalid flagged data is first screened out. Therefore measurements flagged as 'Invalid Data Provider Flags - Scientifically Decreed', 'Extreme Data - Scientifically Non-Feasible', 'Extreme Data - Distributional Outlier', 'Infinite Value', 'Negative Measurement', 'Recurring Value', or measurements are above or below reported (or if not given, documented) limits of detections
#2. numerical data values are averaged 
#3. some qa values which are no longer relevant are dropped (this includes qa flags used to screen out measurements in step 1, and other flags which are no longer relevant for new temporal resolution
#4. other qa values are concatenated 
#5. flags are dropped as no longer relevant

def temporally_grid_data(station_numpy_filename):

    #flags for removing invalid data when averaging
    invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Negative Measurement', 'Zero Measurement', 'No Key Metadata', 'No Altitude Metadata', 'Extreme Data - Scientifically Non-Feasible', 'Extreme Data - Distributional Outlier', 'Extreme Data - Manually Decreed', 'Invalid Data Provider Flags - Scientifically Decreed', 'Recurring Value', 'Hitting LOD', 'Insufficient Measurement Resolution - Reported', 'Insufficient Measurement Resolution - Empirical', 'Below Documented Lower Limit of Detection', 'Below Reported Lower Limit of Detection', 'Above Documented Upper Limit of Detection', 'Above Reported Upper Limit of Detection', 'Methodology Not Mapped', 'Erroneous Primary Sampling', 'Erroneous Sample Preparation', 'Erroneous Measurement Methodology', 'Invalid QA Measurement Methodology']
    invalid_qa_flags = [standardisation_modules.get_standard_qa_flag(invalid_flag_name) for invalid_flag_name in invalid_qa_flag_names]
           
    #nonsensical qa flags when concatenated/modal value is taken, therefore are dropped
    nonsensical_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Negative Measurement', 'Zero Measurement', 'No Key Metadata', 'No Altitude Metadata', 'Assumed Gas Volume', 'Duplicate Time - First Value Kept', 'Overlapping Time - First Value Kept', 'Overlapping Time - Finest Temporal Resolution Kept', 'Extreme Data - Scientifically Non-Feasible', 'Extreme Data - Distributional Outlier', 'Extreme Data - Manually Decreed', 'Invalid Data Provider Flags - Scientifically Decreed', 'Invalid Data Provider Flags - Network Decreed', 'Recurring Value', 'Hitting LOD', 'Insufficient Measurement Resolution - Reported', 'Insufficient Measurement Resolution - Empirical', 'Below Documented Lower Limit of Detection', 'Below Reported Lower Limit of Detection', 'Above Documented Upper Limit of Detection', 'Above Reported Upper Limit of Detection', 'Methodology Not Mapped', 'Erroneous Primary Sampling', 'Erroneous Sample Preparation', 'Erroneous Measurement Methodology', 'Invalid QA Measurement Methodology', 'Daytime', 'Nighttime', 'Weekday', 'Weekend']
    nonsensical_qa_flags = [standardisation_modules.get_standard_qa_flag(invalid_flag_name) for invalid_flag_name in nonsensical_qa_flag_names]
    
    #qa flags to concatenate any instance of in the averaging period into qa flag. 
    concatenate_qa_flag_names = ['Duplicate Station'] 
    concatenate_qa_flags = [standardisation_modules.get_standard_qa_flag(invalid_flag_name) for invalid_flag_name in concatenate_qa_flag_names]
    
    #qa flags to take modal value of across averaging period to write into qa flag.
    modal_qa_flag_names = ['Non-Integer Local Timezone (relative to UTC)', 'High Altitude - Metadata Altitude', 'High Altitude - Metadata Derived', 'High Altitude - ETOPO1', 'High Altitude - Iwahashi Global Landform Classification', 'High Altitude - Meybeck Global Landform Classification', 'Near Coast - Metadata Derived', 'Near Coast - GSFC', 'Rural Station - Lenient Metadata Derived', 'Urban Station - Lenient Metadata Derived', 'Unclassified Station - Lenient Metadata Derived', 'Rural Station - Strict Metadata Derived', 'Urban Station - Strict Metadata Derived', 'Unclassified Station - Strict Metadata Derived', 'Rural Station - Anthrome', 'Urban Station - Anthrome', 'Rural Station - TOAR', 'Urban Station - TOAR', 'Unclassified Station - TOAR', 'Rural Station - Joly-Peuch', 'Unclassified Station - Joly-Peuch', 'Assumed Primary Sampling', 'Assumed Sample Preparation', 'Assumed Measurement Methodology', 'Unknown Primary Sampling Instrument', 'Unknown Measuring Instrument']
    modal_qa_flags = [standardisation_modules.get_standard_qa_flag(invalid_flag_name) for invalid_flag_name in modal_qa_flag_names]
    
    #read in metadata/data
    metadata = read_from_numpy(station_numpy_filename, 'metadata')
    data =     read_from_numpy(station_numpy_filename, 'data')
    
    #get station reference
    station_reference = metadata['station_reference'][0]
    
    #get start and end stamps of data
    start_stamp = data['time'][0]
    end_time = data['time'][-1] 
    end_res = data['temporal_resolution'][-1] 
    if end_res == 'H':
        end_stamp = end_time + 60
    elif end_res == 'D':
        end_stamp = end_time + (60*24)
    elif end_res == 'W':
        end_stamp = end_time + (60*24*7)
    elif end_res == 'M':
        end_dt = fractional_minutes_from_reference_to_datetime(float(end_time))
        end_window_dt = end_dt + relativedelta(months=+1)
        timedelta = ((end_window_dt - end_dt).days)*24*60
        end_stamp = end_time + timedelta
    else:
        end_stamp = end_time + int(end_res)
    
    #if start of first gridded time period is after end of station data range, no valid data in range, therefore do not process further
    if grid_start_minutes[0] >= data['time'][-1]:
        print '%s -- NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'%(station_reference)
        return
    #if end of last gridded time period is before start of station data range, no valid data in range, therefore do not process further
    if grid_start_minutes[-1] <= data['time'][0]:
        print '%s -- NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'%(station_reference)
        return

    #get data keys
    data_keys = list(data.dtype.names)  
    #remove temporal resolution key from list of keys
    data_keys.remove('temporal_resolution')
    
    #create empty numpy dataframe for storing gridded data
    standard_dtype = [(key,data[key].dtype) for key in data_keys]
    gridded_data = np.empty(len(grid_start_minutes[:-1]),dtype=standard_dtype)
    
    #iterate through data_keys to instantiate new numpy dataframe with data
    for kk, key in enumerate(data_keys):
        
        #add gridded time to dataframe
        if key == 'time':
            gridded_data['time'] = grid_start_minutes[:-1] 
        
        #add array of NaNs for np.float32 types  
        elif gridded_data[key].dtype == np.dtype('float32'):
            gridded_data[key] = np.array([np.NaN]*len(grid_start_minutes[:-1]))
        
        #make gridded list of qa codes - with a default of M, i.e. missing measurement
        elif key == 'qa':
            gridded_data[key] = np.array(['M']*len(grid_start_minutes[:-1]))
        
        #make gridded list of flag codes - with a default of N, i.e. Un-flagged data
        elif key == 'flag':
            gridded_data[key] = np.array(['N']*len(grid_start_minutes[:-1]))

    #get valid start and end times  
    #screen out data that has coarser resolution than desired output temporal resolution
    valid_start_times = []
    valid_end_times = []
    valid_timedeltas = []
    valid_inds = []
    for ii, start_time in enumerate(data['time']):
        
        process_flag = False
        
        #get window end time by adding temporal resolution as timedelta
        current_measurement_resolution = data['temporal_resolution'][ii]
        if current_measurement_resolution == 'H':
            timedelta = 60
            if (run_details['temporal_resolution'] == 'H'): 
                process_flag = True
            elif (run_details['temporal_resolution'] == 'D') or (run_details['temporal_resolution'] == 'M'):
                process_flag = True
        elif current_measurement_resolution == 'D':
            timedelta = 60*24
            if (run_details['temporal_resolution'] == 'D'):
                process_flag = True   
            elif (run_details['temporal_resolution'] == 'M'):
                process_flag = True   
        elif current_measurement_resolution == 'W':
            timedelta = (60*24)*7
            if (run_details['temporal_resolution'] == 'M'):
                process_flag = True
        elif current_measurement_resolution == 'M':
            #for monthly timedelta, add relative monthly timedelta from start time, then get difference in minutes between start and end time 
            dt_start = fractional_minutes_from_reference_to_datetime(float(start_time))
            dt_end = dt_start + relativedelta(months=+1)
            timedelta = ((dt_end - dt_start).days)*24*60
            if (run_details['temporal_resolution'] == 'M'):
                process_flag = True
        else:
            timedelta = int(current_measurement_resolution)
            if (run_details['temporal_resolution'] == 'H'):
                if timedelta < 60:
                    process_flag = True
            elif (run_details['temporal_resolution'] == 'D'):
                if timedelta < (24*60):
                    process_flag = True
            elif (run_details['temporal_resolution'] == 'M'):
                if timedelta < (31*24*60):
                    process_flag = True
        
        #get measurement window end time 
        if process_flag == True:
            valid_start_times.append(start_time)
            valid_end_times.append(start_time + timedelta)
            valid_timedeltas.append(timedelta)
            valid_inds.append(ii)

    #check have some valid data after removing coarse resolution data
    if len(valid_inds) == 0:
        print '%s -- AFTER REMOVING COARSE RESOLUTION DATA, NO VALID DATA TO TEMPORALLY GRID'%(station_reference) 
        return
    #next check have some data reported between start_date and end_date - after removing coarse resolutions
    #if start of first gridded time period is after end of station data range, no valid data in range, therefore do not process further
    elif grid_start_minutes[0] >= valid_end_times[-1]:
        print '%s -- AFTER REMOVING COARSE RESOLUTION DATA, NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'%(station_reference)
        return
    #if end of last gridded time period is before start of station data range, no valid data in range, therefore do not process further
    elif grid_start_minutes[-1] <= valid_start_times[0]:
        print '%s -- AFTER REMOVING COARSE RESOLUTION DATA, NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'%(station_reference)
        return
    #do have data to temporally grid
    else:
        print '%s -- TEMPORALLY GRIDDING DATA'%(station_reference)

    #convert cut arrays to numpy arrays
    valid_start_times = np.array(valid_start_times, dtype=np.uint32)
    valid_end_times = np.array(valid_end_times, dtype=np.uint32)
    valid_timedeltas = np.array(valid_timedeltas , dtype=np.uint32)
    valid_inds = np.array(valid_inds , dtype=np.uint32)
    
    #flag that determines if have valid data to write for station to numpy .npz file
    data_to_write = False

    #get index of first window which contains measurements, and iterate through windows from there 
    window_start_index = bisect.bisect(grid_start_minutes, valid_start_times[0])-1
    
    #get first valid start datetime of window 
    current_dt = fractional_minutes_from_reference_to_datetime(float(grid_start_minutes[window_start_index]))
    
    #initialise variables for determining is new day/week/month
    current_year = current_dt.year
    current_month = current_dt.month
    current_week = current_dt.isocalendar()[1]
    current_day = current_dt.day
    
    #initialise variables for evaluating temporal representation across periods
    temporal_representation_string = ''
    n_minutes_represented = 0
    native_gaps = []
    if run_details['temporal_resolution'] == 'H':
        temporal_representation_string = temporal_representation_string + standardisation_modules.get_standard_qa_flag('Not Hourly Representative')
        daily_n_minutes_represented = 0
        daily_gaps = []
        daily_inds = []
        daily_open_gap = False
    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
        temporal_representation_string = temporal_representation_string + (standardisation_modules.get_standard_qa_flag('Not Diurnally Representative') + standardisation_modules.get_standard_qa_flag('Not Weekly Representative'))
        weekly_n_minutes_represented = 0
        weekly_gaps = []
        weekly_inds = []
        weekly_open_gap = False
        monthly_n_minutes_represented = 0
        monthly_gaps = []
        monthly_inds = []
        monthly_open_gap = False
    temporal_representation_string = temporal_representation_string + (standardisation_modules.get_standard_qa_flag('Not Monthly Representative') + standardisation_modules.get_standard_qa_flag('Not Annually Representative'))
    annual_n_minutes_represented = 0
    annual_gaps = []
    annual_inds = []
    annual_open_gap = False
    
    #set all qa for windows before first valid datetime to be not temporally representative for appropriate periods
    for qa_xx in range(window_start_index):
        gridded_data['qa'][qa_xx] = gridded_data['qa'][qa_xx] + temporal_representation_string

    #iterate through gridded time
    for ii, window_start_minute in enumerate(grid_start_minutes[window_start_index:-1]):
        
        write_out = False
        actual_ii = window_start_index+ii
        window_end_minute = grid_start_minutes[actual_ii+1]
    
        #print '**NEW WINDOW**'
        #print fractional_minutes_from_reference_to_datetime(float(window_start_minute)), fractional_minutes_from_reference_to_datetime(float(window_end_minute))
    
        #if last of valid end times is <= left edge of window, will be no more valid times in window, so break out of iteration
        if valid_end_times[-1] <=  window_start_minute:
            
            #set all qa after last valid datetime to be not temporally representative for appropriate periods
            for qa_xx in range(actual_ii,len(grid_start_minutes[:-1])):
                gridded_data['qa'][qa_xx] = gridded_data['qa'][qa_xx] + temporal_representation_string
            break
            
        #if left edge of window is <= first valid start time, set start_window_i to 0
        if window_start_minute <= valid_start_times[0]:
            start_window_i = 0
            left_aligned = True
        #if left edge of window is > last valid start time, set start_window_i to last valid index
        elif window_start_minute > valid_start_times[-1]:
            start_window_i = len(valid_start_times)-1
        #otherwise, check if first valid start time aligns with left edge of window
        else:
            start_window_i = bisect.bisect_left(valid_start_times,window_start_minute)
            #is start time aligned with start of window?
            if window_start_minute != valid_start_times[start_window_i]:
                #if not aligned, minus 1 to index correctly
                start_window_i = start_window_i - 1
                left_aligned = False
            else:
                left_aligned = True
        
        #if right edge of window is >= last valid end time, set end_window_i to number of valid times
        if window_end_minute >= valid_end_times[-1]:
            end_window_i = len(valid_end_times)
            right_aligned = True
        #otherwise, check if last valid end time aligns with right edge of window
        else:
            end_window_i = bisect.bisect_left(valid_end_times,window_end_minute)
            #is end time aligned with end of window?
            if window_end_minute != valid_end_times[end_window_i]:
                #if not aligned, plus 1 to index correctly
                end_window_i = end_window_i + 1
                right_aligned = False
            else:
                right_aligned = True                           

        #get difference between relevant indices
        window_index_diff = end_window_i - start_window_i
        #just 1 relevant index
        if window_index_diff == 0:
            window_indices = np.array([start_window_i])
            #print '1 INDEX'
            #for sss,eee in zip(valid_start_times[window_indices],valid_end_times[window_indices]):
            #    print fractional_minutes_from_reference_to_datetime(float(sss)), fractional_minutes_from_reference_to_datetime(float(eee))
        #multiple relevant indices
        else:
            window_indices = np.arange(start_window_i, end_window_i)

        #get fractional overlaps on edges of windows (if exist)
        if len(window_indices) > 1:
            left_overlap = np.NaN
            right_overlap = np.NaN
            
            #print 'INITIAL TIMES:'
            #for sss,eee in zip(valid_start_times[window_indices],valid_end_times[window_indices]):
            #    print fractional_minutes_from_reference_to_datetime(float(sss)), fractional_minutes_from_reference_to_datetime(float(eee))
            
            #Determine if the first measurement period end time is within the measurement window, if not slice off first index
            if valid_end_times[window_indices[0]] <= window_start_minute: 
                window_indices = window_indices[1:]
            
            #Determine if the last measurement period start time is within the measurement window, if not slice off last index
            if valid_start_times[window_indices[-1]] >= window_end_minute: 
                window_indices = window_indices[:-1]
        
            #if have 0 window indices left, it is because the measurement window is in a void between measurements
            if len(window_indices) == 0:
                n_minutes_represented = 0
                native_gap = window_end_minute - window_start_minute
                if run_details['temporal_resolution'] == 'H':
                    daily_gaps.append(native_gap)
                    daily_open_gap = True
                if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                    weekly_gaps.append(native_gap) 
                    monthly_gaps.append(native_gap) 
                    weekly_open_gap = True
                    monthly_open_gap = True
                annual_gaps.append(native_gap)
                annual_open_gap = True
                continue
        
            if len(window_indices) > 1:
                #if the left edge of measurement window does align with the end/start of a measurement period, determine why this is.
                if left_aligned == False:

                    #Determine if the first measurement period overlaps the left window edge
                    if (valid_start_times[window_indices[0]] - window_start_minute) < 0:
                        #get number of minutes of measurement period overlapping measurement window on left edge
                        left_overlap = valid_end_times[window_indices[0]] - window_start_minute
                
                #if the right edge of measurement window does align with the end/start of a measurement period, determine why this is.
                if right_aligned == False:
            
                    #Determine if the last measurement period overlaps the right window edge
                    if (window_end_minute - valid_end_times[window_indices[-1]]) < 0:
                        #get number of minutes of measurement period overlapping measurement window on right edge
                        right_overlap = window_end_minute - valid_start_times[window_indices[-1]]

            #print 'PROCESSED TIMES:'
            #for sss,eee in zip(valid_start_times[window_indices],valid_end_times[window_indices]):
            #    print fractional_minutes_from_reference_to_datetime(float(sss)), fractional_minutes_from_reference_to_datetime(float(eee))
                
        #get indices of measurements within window, respective to full data arrays
        if len(window_indices) > 0:
            data_respective_window_indices = valid_inds[window_indices]
            
        #if just have 1 index, get number of minutes represented in window and maximum temporal gap (if data is valid with regards to qa flags)
        if len(window_indices) == 1:
            #if have 1 index will always write out (unless explicitly choose to filter by qa checks before)
            write_out = True
            #print data['qa'][data_respective_window_indices], data_respective_window_indices
            valid_data_respective_window_indices = data_respective_window_indices[np.array([all(x not in qa_str for x in invalid_qa_flags) for qa_str in data['qa'][data_respective_window_indices]])]
            if len(valid_data_respective_window_indices) > 0:
                relevant_start_time = valid_start_times[window_indices][0] 
                relevant_end_time = valid_end_times[window_indices][0]
                #get n minutes represented
                n_minutes_represented = relevant_end_time - window_start_minute
                #add n_minutes_represented for other temporal resolutions
                if run_details['temporal_resolution'] == 'H':
                    daily_n_minutes_represented = daily_n_minutes_represented + n_minutes_represented
                if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                    weekly_n_minutes_represented = weekly_n_minutes_represented + n_minutes_represented
                    monthly_n_minutes_represented = monthly_n_minutes_represented + n_minutes_represented
                annual_n_minutes_represented = annual_n_minutes_represented + n_minutes_represented
                
                #get max temporal gap across window (native output resolution)
                native_gaps = [(window_end_minute-window_start_minute) - n_minutes_represented]
                #append/process gaps across other temporal resolutions
                left_edge_gap = relevant_start_time - window_start_minute
                right_edge_gap = window_end_minute - relevant_end_time
                if left_edge_gap > 0:
                    if run_details['temporal_resolution'] == 'H':
                        if daily_open_gap == True:
                            daily_gaps[-1] = daily_gaps[-1] + left_edge_gap
                        else:
                            daily_gaps.append(left_edge_gap)
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        if weekly_open_gap == True:
                            weekly_gaps[-1] = weekly_gaps[-1] + left_edge_gap
                        else:
                            weekly_gaps.append(left_edge_gap)
                        if monthly_open_gap == True:
                            monthly_gaps[-1] = monthly_gaps[-1] + left_edge_gap
                        else:
                            monthly_gaps.append(left_edge_gap)
                    if annual_open_gap == True:
                        annual_gaps[-1] = annual_gaps[-1] + left_edge_gap 
                    else:
                        annual_gaps.append(left_edge_gap)
                           
                else:
                    if run_details['temporal_resolution'] == 'H':
                        daily_open_gap = False
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        weekly_open_gap = False
                        monthly_open_gap = False
                    annual_open_gap = False
                
                if right_edge_gap > 0:
                    if run_details['temporal_resolution'] == 'H':
                        daily_gaps.append(right_edge_gap)
                        daily_open_gap = True
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        weekly_gaps.append(right_edge_gap)
                        monthly_gaps.append(right_edge_gap)
                        weekly_open_gap = True
                        monthly_open_gap = True
                    annual_gaps.append(right_edge_gap)
                    annual_open_gap = True
                else:
                    if run_details['temporal_resolution'] == 'H':
                        daily_open_gap = False
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        weekly_open_gap = False
                        monthly_open_gap = False
                    annual_open_gap = False
        
            #no valid data
            else:
                #if choose to explicitly filter by qa checks, and have no valid data in window, do not write out
                if filter_output == True:
                    write_out = False
                n_minutes_represented = 0
                native_gaps = [window_end_minute-window_start_minute]
                if run_details['temporal_resolution'] == 'H':
                    daily_gaps.extend(native_gaps)
                    daily_open_gap = True
                if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                    weekly_gaps.extend(native_gaps)
                    monthly_gaps.extend(native_gaps) 
                    weekly_open_gap = True
                    monthly_open_gap = True
                annual_gaps.extend(native_gaps)
                annual_open_gap = True
                             
        #if are averaging data within window (i.e. more than 1 measurement, screen data to remove invalid data
        #also calculate number of minutes represented in window and maximum temporal gap
        elif len(window_indices) > 1:
        
            #weight data by timedeltas for averaging different temporal resolution data 
            window_weights = valid_timedeltas[window_indices]
            #modify weights on left and right edges to adjust for actual time sampled within window, if measurement overlaps edge
            if pd.isnull(left_overlap) == False:
                window_weights[0] = left_overlap
            if pd.isnull(right_overlap) == False:
                window_weights[-1] = right_overlap
        
            #cut arrays solely for valid values
            valids = np.array([all(x not in qa_str for x in invalid_qa_flags) for qa_str in data['qa'][data_respective_window_indices]])
            window_indices = window_indices[valids]
            window_weights = window_weights[valids]
            valid_data_respective_window_indices = data_respective_window_indices[valids]

            if len(valid_data_respective_window_indices) > 0:
                #have valid data after qa flags, therefore can write out
                write_out = True
                
                relevant_start_times = valid_start_times[window_indices] 
                relevant_end_times = valid_end_times[window_indices]
                
                #get n minutes represented
                n_minutes_represented = np.sum(window_weights)
                
                #add n_minutes_represented for other temporal resolutions
                if run_details['temporal_resolution'] == 'H':
                    daily_n_minutes_represented = daily_n_minutes_represented + n_minutes_represented
                if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                    weekly_n_minutes_represented = weekly_n_minutes_represented + n_minutes_represented
                    monthly_n_minutes_represented = monthly_n_minutes_represented + n_minutes_represented
                annual_n_minutes_represented = annual_n_minutes_represented + n_minutes_represented
            
                #get maximum temporal gap across window between valid observations
                native_gaps = []
                #deal with open gaps on left edge of window
                left_edge_gap = relevant_start_times[0] - window_start_minute
                if left_edge_gap > 0:
                    native_gaps.append(left_edge_gap)
                    if run_details['temporal_resolution'] == 'H':
                        if daily_open_gap == True:    
                            daily_gaps[-1] = daily_gaps[-1] + left_edge_gap
                        else:
                            daily_gaps.append(left_edge_gap)
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        if weekly_open_gap == True: 
                            weekly_gaps[-1] = weekly_gaps[-1] + left_edge_gap
                        else:
                            weekly_gaps.append(left_edge_gap)
                        if monthly_open_gap == True:
                            monthly_gaps[-1] = monthly_gaps[-1] + left_edge_gap
                        else:
                            monthly_gaps.append(left_edge_gap)
                    if annual_open_gap == True:
                        annual_gaps[-1] = annual_gaps[-1] + left_edge_gap
                    else:
                        annual_gaps.append(left_edge_gap)

                else:
                    if run_details['temporal_resolution'] == 'H':
                        daily_open_gap = False
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        weekly_open_gap = False
                        monthly_open_gap = False
                    annual_open_gap = False
                    
                #deal with open gaps in centre of window
                for obs_start,obs_end in zip(relevant_start_times[1:],relevant_end_times[:-1]):
                    gap = obs_start - obs_end
                    #print gap, obs_start, obs_end
                    if gap > 0:
                        native_gaps.append(gap)
                        if run_details['temporal_resolution'] == 'H':
                            daily_gaps.append(gap)
                        if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                            weekly_gaps.append(gap)
                            monthly_gaps.append(gap)
                        annual_gaps.append(gap)
                                    
               #deal with open gaps on right edge of window
                right_edge_gap = window_end_minute - relevant_end_times[-1]
                if right_edge_gap > 0:
                    native_gaps.append(right_edge_gap)
                    if run_details['temporal_resolution'] == 'H':
                        daily_gaps.append(right_edge_gap)
                        daily_open_gap = True
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        weekly_gaps.append(right_edge_gap)
                        monthly_gaps.append(right_edge_gap)
                        weekly_open_gap = True
                        monthly_open_gap = True
                    annual_gaps.append(right_edge_gap)
                    annual_open_gap = True
                else:
                    if run_details['temporal_resolution'] == 'H':
                        daily_open_gap = False
                    if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                        weekly_open_gap = False
                        monthly_open_gap = False
                    annual_open_gap = False
                    
            #no valid data
            else:
                n_minutes_represented = 0
                native_gaps = [window_end_minute - window_start_minute]
                if run_details['temporal_resolution'] == 'H':
                    daily_gaps.extend(native_gaps)
                    daily_open_gap = True
                if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                    weekly_gaps.extend(native_gaps) 
                    monthly_gaps.extend(native_gaps) 
                    weekly_open_gap = True
                    monthly_open_gap = True
                annual_gaps.extend(native_gaps) 
                annual_open_gap = True
        
        #determine if next datetime is a new day/week/month
        #process next start window datetime
        next_dt = fractional_minutes_from_reference_to_datetime(float(window_end_minute))
    
        #is it a new year
        if (next_dt.year != current_year):
            new_year = True
        else:
            annual_inds.append(actual_ii)
            new_year = False
        #is it a new month?
        if (next_dt.year != current_year) or (next_dt.month != current_month):
            new_month = True
        else:
            monthly_inds.append(actual_ii)
            new_month = False
        #is it a new week?
        if (next_dt.year != current_year) or (next_dt.isocalendar()[1] != current_week):
            new_week = True
        else:
            weekly_inds.append(actual_ii)
            new_week = False
        #is it a new day?
        if (next_dt.year != current_year)  or (next_dt.month != current_month) or (next_dt.day != current_day):
            new_day = True
        else:
            daily_inds.append(actual_ii)
            new_day = False

        #evaluate native temporal resolution representation (every pass - if have valid data)
        gridded_data['qa'] = quality_control_modules.check_representation(n_minutes_represented, native_gaps, run_details['temporal_resolution'], gridded_data['qa'], [actual_ii], current_dt)                
        #reset native variables
        n_minutes_represented = 0
        native_gaps = []
            
        #if hourly output resolution, calculate representation of previous day (if next index is new day )
        if run_details['temporal_resolution'] == 'H':
            if new_day == True:
                if len(daily_inds) > 0:
                    gridded_data['qa'] = quality_control_modules.check_representation(daily_n_minutes_represented, daily_gaps, 'D', gridded_data['qa'], daily_inds, current_dt)
                #reset daily variables
                daily_n_minutes_represented = 0
                daily_gaps = []
                daily_inds = []
                daily_open_gap = False
                   
            #if hourly/daily output resolution, calculate representation of previous week (if next index is new week)
            if (run_details['temporal_resolution'] == 'H') or (run_details['temporal_resolution'] == 'D'):
                if new_week == True:
                    if len(weekly_inds) > 0:
                        gridded_data['qa'] = quality_control_modules.check_representation(weekly_n_minutes_represented, weekly_gaps, 'W', gridded_data['qa'], weekly_inds, current_dt)
                    #reset weekly variables
                    weekly_n_minutes_represented = 0
                    weekly_gaps = []
                    weekly_inds = []
                    weekly_open_gap = False
                   
                #calculate representation of previous month (if next index is new month)     
                if new_month == True:
                    if len(monthly_inds) > 0:
                        gridded_data['qa'] = quality_control_modules.check_representation(monthly_n_minutes_represented, monthly_gaps, 'M', gridded_data['qa'], monthly_inds, current_dt)
                    #reset monthly variables
                    monthly_n_minutes_represented = 0
                    monthly_gaps = []
                    monthly_inds = []
                    monthly_open_gap = False
                    
            #for all output resolutions, calculate representation of previous year (if next index is new year)
            if new_year == True:
                if len(annual_inds) > 0:
                    gridded_data['qa'] = quality_control_modules.check_representation(annual_n_minutes_represented, annual_gaps, 'A', gridded_data['qa'], annual_inds, current_dt)
                #reset annual variables
                annual_n_minutes_represented = 0
                annual_gaps = []
                annual_inds = []
                annual_open_gap = False
            
        #update current year/month/week/day (for next pass)
        current_year = next_dt.year
        current_month = next_dt.month
        current_week = next_dt.isocalendar()[1]
        current_day = next_dt.day
        current_dt = next_dt
            
        #proceed if have valid indices for window after screening scientifically invalid data (when have > 1 window indices), or have chosen to filter by QA flags (when have 1 index)
        if write_out == True:
    
            data_to_write = True
        
            #iterate through data variables and process data onto desired temporal resolution grid
            for key in data_keys:

                #variables of type: float32
                if data[key].dtype == np.dtype('float32'):
                
                    #if are averaging data into desired resolution, take weighted average
                    if len(window_indices) > 1:
                        window_weighted_average = np.average(data[key][valid_data_respective_window_indices], weights=window_weights)
                        gridded_data[key][actual_ii] = window_weighted_average
                    else:
                        gridded_data[key][actual_ii] = data[key][data_respective_window_indices][0]

                #qa codes
                elif key == 'qa':
                
                    #if have more than 1 measurement in window, need to deal with multiple qa flags
                    #this is done in 2 ways, firstly dropping qa codes which are no longer relevant when taken across an averaged period (most of which have already been applied to screen invalid data)
                    #secondly, for some qa codes are retained if they are more frequently reported across period than not, otherwise they are dropped
                    #finally, concatenate all unique qa codes which remain in the period into 1 unique qa code. There are some codes that have are not delat with by first 2 steps, therefore if solely one instance of code in period, it is retained (i.e. if one of points is flagged as urban influenced, all will be flagged as urban influenced.)  
                    if len(window_indices) > 1:
                
                        #if averaging data, concatenate all qa strings into 1 string for all of time period
                        concat_qa = ''.join(data[key][valid_data_respective_window_indices])
                    
                        #split concatenated string into a list of qa codes
                        split_qa = re.findall('[A-Z][^A-Z]*', concat_qa)
                    
                        #make list of unique qa codes   
                        split_unique_qa = list(set(split_qa))
                    
                        #some qa flags in list no longer make sense when taking average over period, therefore drop them
                        split_unique_qa = [x for x in split_unique_qa if x not in nonsensical_qa_flags]
                        
                        #for some qa codes need to check if they are more frequently reported or not across the period, if more frequently not reported, drop the code
                        for modal_flag in modal_qa_flags:
                            #first only proceed if flag is actually at all reported in the window
                            if modal_flag in split_unique_qa:
                                reported_count = 0
                                not_reported_count = 0
                                for ww, qa_code in enumerate(data[key][valid_data_respective_window_indices]):
                                    if modal_flag in qa_code: 
                                        reported_count += window_weights[ww] 
                                    else:
                                        not_reported_count += window_weights[ww] 
                                if not_reported_count > reported_count:
                                    split_unique_qa.remove(modal_flag)

                        #put list of unique relevant, processed qa codes back together into 1 string
                        unique_qa = ''.join(split_unique_qa)
                    
                        #append new unique relevant qa code for averaged period
                        gridded_data[key][actual_ii] = unique_qa
                    
                    else:
                        gridded_data[key][actual_ii] = data[key][data_respective_window_indices][0]
                
                #data flags
                elif key == 'flag':
                
                    #if have more than 1 measurement in window, then discard flag as will have already taken account of them (through associated qa codes) and do not make sense as an average over a period
                    if len(window_indices) > 1:
                
                        #append valid data string as will have passed invalid flag screening
                        gridded_data[key][actual_ii] = 'V'
                    
                    else:  
                        gridded_data[key][actual_ii] = data[key][data_respective_window_indices][0]

    #proceed to write out to station specific numpy file
    if data_to_write == True:
    
        #create temporary numpy file for station 
        write_station_specific_temporary_bsc_numpy_by_month(metadata, gridded_data, start_stamp, end_stamp)
        
    #otherwise no valid data has been found
    else:
        print 'NO VALID DATA FOUND AFTER QA FLAG CHECKS'
        
    return 

###------------------------------------------------------------------------------------###

#write temporary .npz file, month by month, in desired temporal range for specific station 

def write_station_specific_temporary_bsc_numpy_by_month(metadata,data,start_stamp,end_stamp):
    
    #get station reference
    station_reference = metadata['station_reference'][0]

    print '%s -- WRITING TO MONTHLY STATION SPECIFIC TEMPORALLY GRIDDED .npz FILES'%(station_reference)
    
    #get metadata updata stamp variable
    update_metadata_stamp = metadata['meta_update_stamp']
    
    #some metadata variables make no sense when taken as a modal value across the month (i.e. timestamps)
    #therefore do write/process variables of this type 
    #remove from metadata record array
    metadata = rf.drop_fields(metadata, ['meta_update_stamp','data_download_stamp'])
    
    #get metadata/data keys
    metadata_keys = list(metadata.dtype.names)
    data_keys =     list(data.dtype.names)
    
    #get start and end datetimes of raw data
    start_dt = fractional_minutes_from_reference_to_datetime(float(start_stamp))
    end_dt = fractional_minutes_from_reference_to_datetime(float(end_stamp))
    
    #get data start_year/start_month and end_year/end month  
    data_start_year = start_dt.year
    data_start_month = start_dt.month
    data_end_year = end_dt.year
    data_end_month = end_dt.month
    data_end_day = end_dt.day
    data_end_hour = end_dt.hour
    #adjust data end year (if end date is 20150101 0000 for example, the end year should be 2014, not 2015)
    if (data_end_month == 1) & (data_end_day == 1) & (data_end_hour == 0):
        data_end_year = data_end_year - 1
        data_end_month = 12
    #adjust data end month (if end date is 20150301 0000 for example, the end month should be 2, not 3)
    elif (data_end_day == 1) & (data_end_hour == 0):
        data_end_month = data_end_month - 1
    
    #create list of years to process from provided start_date and end_date
    desired_start_year = int(run_details['start_date'][:4])
    desired_end_year = int(run_details['end_date'][:4])
    desired_start_month = int(run_details['start_date'][4:])
    desired_end_month = int(run_details['end_date'][4:])
    process_years = np.arange(desired_start_year, desired_end_year+1)

    #cut list of years to process for years which are >= data_start_year and <= data_end_year
    process_years = process_years[process_years >= data_start_year]
    process_years = process_years[process_years <= data_end_year]
    
    #only process months which are 
    
    #loop through years and months to process, and write to monthly station specific .npz files
    #iterate through process_years
    for year in process_years:
        
        #create list of months to process
        
        #if year == data start year
        if year == desired_start_year:
            process_months = np.arange(desired_start_month, 13)
            
        #if year == desired end year
        elif year == desired_end_year:
            process_months = np.arange(1,desired_end_month)
        
        #full year of months otherwise
        else:
            process_months = np.arange(1,13)
        
        #if in year that data finishes or ends then cut list of months to process for months which are >= data_start_month and <= data_end_month
        if year == data_start_year:
            process_months = process_months[process_months>=data_start_month]
            
        elif year == data_end_year:
            process_months = process_months[process_months<=data_end_month]
        
        #iterate through process_months
        for month in process_months:
            
            if month < 10:
                month_string = '0'+str(month)
            else:
                month_string = str(month)
            
            #get minutes of start and end of monthly window
            start_dt = datetime.datetime(year,month,01,00,00)
            end_dt = start_dt + relativedelta(months=+1)
            start_window_minutes = datetime_to_fractional_minutes_from_reference(start_dt)
            end_window_minutes = datetime_to_fractional_minutes_from_reference(end_dt)
        
            #cut data,metadata for monthly period
            #after gridding, data times are guaranteed to align to window edges
            start_data_i = bisect.bisect_left(data['time'], start_window_minutes)
            end_data_i = bisect.bisect_left(data['time'], end_window_minutes)
            cut_data = data[start_data_i:end_data_i] 
            #check you have some valid data for the month (i.e. not all NaNs), if not continue to next month
            if all(pd.isnull(cut_data[parameter_details['bsc_parameter_name']])) == True:
                continue
            #make time array (hours/days/months) from start of month to end of month
            time_array = np.arange(len(cut_data['time']), dtype=cut_data['time'].dtype)
            #overwrite old time array
            cut_data['time'] = time_array
                            
            #meta stamp times are not guaranteed however to align with window edges
            start_meta_i = bisect.bisect(update_metadata_stamp,start_window_minutes)-1
            #if meta starts partially through window, start index will be returned as -1, change this to 0 (as want all from start)
            if start_meta_i < 0:
                start_meta_i = 0
            end_meta_i = bisect.bisect_left(update_metadata_stamp,end_window_minutes)
            cut_metadata = metadata[start_meta_i:end_meta_i] 
            cut_update_metadata_stamp = update_metadata_stamp[start_meta_i:end_meta_i] 
            
            #get modal metadata in month
            #if have more than 1 row of metadata in month, at least 1 field has multiple metadata values, take modal value across month for each field (by most minutes occurring)                
            if cut_metadata.shape[0] > 1:
                
                #create numpy dataframe to place modal metadata in month
                #create lists for standardising data formats of the input data dictionary
                standard_dtype = [(key,metadata[key].dtype) for key in metadata_keys]
                #create numpy dataframe to store standardised forms of variables inside data dictionary
                modal_metadata = np.empty(1, dtype=standard_dtype)
                
                for key in metadata_keys:

                    #metadata values for field
                    metadata_values = cut_metadata[key]
                        
                    #take mode of weighted metadata (by number of minutes that it is relevant for) that does not equal NaN
                    #if are multiple items with same number of instances, the lowest sorted instance will be returned (i.e. 'a' rather 'b'; 1 rather than 2)
                    weighted_meta = {}
                    for xx, metadata_value in enumerate(metadata_values):
                        #last index
                        if (xx+1) == len(metadata_values):
                            minutes_represented = end_window_minutes - cut_update_metadata_stamp[xx]
                        else:
                            minutes_represented = cut_update_metadata_stamp[xx+1] - cut_update_metadata_stamp[xx]
        
                        if pd.isnull(metadata_value) == False:
                            if metadata_value in weighted_meta.keys():
                                weighted_meta[metadata_value] = weighted_meta[metadata_value]+minutes_represented
                            else:
                                weighted_meta[metadata_value] = minutes_represented
    
                    #if have no valid metadata, modal value is np.NaN
                    if len(weighted_meta) == 0:
                         modal_value = np.NaN
                    else:
                        counter_instance = Counter(weighted_meta)
                        modal_value = counter_instance.most_common(1)[0][0]
                    
                    #append model value to new metadata frame
                    modal_metadata[key] = modal_value
                
                modal_metadata = modal_metadata[0]           

            #only 1 value in month for each metadata variable, so existing array is model metadata array
            else:
                modal_metadata = cut_metadata[0]
                
            #write to numpy .npz file
            np.savez_compressed('/esarchive/obs/ghost/%s/%s/%s/temporary/%s/%s/%s.npz'%(run_details['network'],desired_resolution_path,parameter_details['bsc_parameter_name'],year,month_string,station_reference), data=cut_data, metadata=modal_metadata)

    return 
        
###------------------------------------------------------------------------------------###

#gather all station specific .npz files in list of lists by month

def get_numpy_files_per_month():
        
    files_to_output = []
        
    years = np.arange(int(run_details['start_date'][:4]),int(run_details['end_date'][:4])+1)
    
    #iterate through years
    for year in years:
    
        #is it first process year?
        if year == years[0]:
            months = np.arange(int(run_details['start_date'][4:]), 13)
                    
        #is it last process year?
        elif year == years[-1]:
            months = np.arange(1,int(run_details['end_date'][4:]))
                    
        #full year of months otherwise
        else:
            months = np.arange(1,13)
    
        #iterate through months
        for month in months:
            
            if month < 10:
                month_string = '0'+str(month)
                yearmonth_str = str(year)+month_string
            else:
                month_string = str(month)
                yearmonth_str = str(year)+month_string
            
            #check if have station files in year/month directory
            year_month_files = sorted(glob.glob('/esarchive/obs/ghost/%s/%s/%s/temporary/%s/%s/*.npz'%(run_details['network'],desired_resolution_path,parameter_details['bsc_parameter_name'],year,month_string)))  
            
            #if so, then append to monthly list of files
            if len(year_month_files) > 0:
                files_to_output.append([yearmonth_str, year_month_files])
                 
    return files_to_output

###------------------------------------------------------------------------------------###

#write out final merged netcdf form all station specific .npz files for a specific month

def write_final_merged_bsc_netcdf_by_month(zipped_list):
    
    #unzip variables    
    yearmonth_str, year_month_files = zipped_list[0],zipped_list[1]
    
    #get year and month
    year = yearmonth_str[:4]
    month = yearmonth_str[-2:]
    
    #dictionary to map python types to netcdf types
    type_map = {np.dtype('uint32'):'u4', np.dtype('O'):'S1', np.dtype('float32'):'f4', np.dtype('float64'):'f8'}
    
    print 'WRITING FINAL MERGED GRIDDED NETCDF IN %s'%(yearmonth_str)

    #create netcdf frame
    root_grp = Dataset('/esarchive/obs/ghost/%s/%s/%s/%s_%s.nc'%(run_details['network'],desired_resolution_path,parameter_details['bsc_parameter_name'],parameter_details['bsc_parameter_name'],yearmonth_str), 'w', format="NETCDF4")    

    #do not pre-fill arrays will default fill values
    root_grp.set_fill_off()

    #file contents
    root_grp.title = 'Surface %s data in the %s network in %s-%s.'%(parameter_details['long_parameter_name'],run_details['network'], year, month)
    root_grp.institution = 'Barcelona Supercomputing Center'
    root_grp.source = 'Surface observations'
    root_grp.creator_name = 'Dene R. Bowdalo'
    root_grp.creator_email = 'dene.bowdalo@bsc.es'
    root_grp.conventions = 'CF-1.7'
    root_grp.data_version= '1.0'

    #iterate through station year/month files and merge to one netcdf
    for ii, station_file in enumerate(year_month_files):
        
        monthly_metadata = read_from_numpy(station_file, 'metadata')
        monthly_data =     read_from_numpy(station_file, 'data')

        #station reference
        station_reference = monthly_metadata['station_reference']

        #get metadata/data keys 
        metadata_keys = monthly_metadata.dtype.names
        data_keys =     monthly_data.dtype.names

        #if first station, create some variables for writing
        if ii == 0:

            #dicts with standard formats for metadata/data
            metadata_format_dict = standardisation_modules.create_standardised_metadata_frame()
            data_format_dict     = standardisation_modules.create_standardised_data_frame()
            
            #get time array
            time_array = monthly_data['time'][:]
            
            #netcdf dimensions
            root_grp.createDimension('time', len(time_array))
            root_grp.createDimension('station', len(year_month_files))
            root_grp.createDimension('strlen%s'%(run_details['max_character_limit_short']), run_details['max_character_limit_short'])
            root_grp.createDimension('strlen%s'%(run_details['max_character_limit_long']), run_details['max_character_limit_long'])

            #set variables for data parameters
            data_vars = []
            for data_key in data_keys:
                current_data_type = type_map[monthly_data[data_key].dtype]

                if current_data_type == 'S1':
                    if 'short' in data_format_dict[data_key]['string_format']: 
                        var = root_grp.createVariable(data_key, current_data_type, ('time','station','strlen%s'%(run_details['max_character_limit_short'])), zlib=True, complevel=4, chunksizes=[root_grp.dimensions['time'].size, root_grp.dimensions['station'].size, root_grp.dimensions['strlen%s'%(run_details['max_character_limit_short'])].size])
                    elif 'long' in data_format_dict[data_key]['string_format']: 
                        var = root_grp.createVariable(data_key, current_data_type, ('time','station','strlen%s'%(run_details['max_character_limit_long'])), zlib=True, complevel=4, chunksizes=[root_grp.dimensions['time'].size, root_grp.dimensions['station'].size, root_grp.dimensions['strlen%s'%(run_details['max_character_limit_long'])].size])
                else:
                    if data_key == 'time':
                        var = root_grp.createVariable(data_key, current_data_type, ('time'), zlib=True, complevel=4, chunksizes=[root_grp.dimensions['time'].size])
                    else:
                        var = root_grp.createVariable(data_key, current_data_type, ('time','station'), zlib=True, complevel=4, chunksizes=[root_grp.dimensions['time'].size, root_grp.dimensions['station'].size])

                #set variable attributes
                var.standard_name = data_format_dict[data_key]['standard_name']
                var.long_name = data_format_dict[data_key]['long_name']
                var.units = data_format_dict[data_key]['units']
                var.description = data_format_dict[data_key]['description']

                #variable specific attributes
                if data_key == 'time':
                    var.units = '%s since %s-%s-01 00:00:00'%(descriptive_res,year,month)
                    var.description = 'Time in %s since %s-%s-01 00:00 UTC. Time given refers to the start of the time window the measurement is representative of (temporal resolution).'%(descriptive_res,year,month)
                    var.axis = 'T'
                    var.calendar = 'standard'
                    var.tz = 'UTC'

            #set variables for metadata parameters
            for metadata_key in metadata_keys:

                current_data_type = type_map[monthly_metadata[metadata_key].dtype]

                if current_data_type == 'S1':
                    if 'short' in metadata_format_dict[metadata_key]['string_format']:
                        var = root_grp.createVariable(metadata_key, current_data_type, ('station','strlen%s'%(run_details['max_character_limit_short'])), zlib=True, complevel=4, chunksizes=[root_grp.dimensions['station'].size, root_grp.dimensions['strlen%s'%(run_details['max_character_limit_short'])].size])
                    elif 'long' in metadata_format_dict[metadata_key]['string_format']: 
                        var = root_grp.createVariable(metadata_key, current_data_type, ('station','strlen%s'%(run_details['max_character_limit_long'])), zlib=True, complevel=4, chunksizes=[root_grp.dimensions['station'].size, root_grp.dimensions['strlen%s'%(run_details['max_character_limit_long'])].size])
                else:
                    var = root_grp.createVariable(metadata_key, current_data_type, ('station',), zlib=True, complevel=4, chunksizes=[root_grp.dimensions['station'].size])

                #set variable attributes
                var.standard_name = metadata_format_dict[metadata_key]['standard_name']
                var.long_name = metadata_format_dict[metadata_key]['long_name']
                var.units = metadata_format_dict[metadata_key]['units']
                var.description = metadata_format_dict[metadata_key]['description']

                #variable specific attributes
                if metadata_key == 'longitude':
                    var.axis = 'X'
                elif metadata_key == 'latitude':
                    var.axis = 'Y'

        #-----------------------------------------------------------#
        #write out metadata/data

        #write out data
        for data_key in data_keys:

            #convert list of strings to character array
            #first need to convert np.object type to np.string array
            if root_grp[data_key].dtype == 'S1':
                char_array = stringtochar(monthly_data[data_key].astype(np.str))
                current_max_char = char_array.shape[1]
                     
                #if characters of any of metadata exceeds character limit, limit it to max characters
                if current_max_char > root_grp[data_key].shape[2]:
                    char_array = char_array[:,:root_grp[data_key].shape[2]]
                    current_max_char = char_array.shape[1]
            
                root_grp[data_key][:,ii,:current_max_char] = char_array
            else:
                if data_key == 'time':
                    root_grp[data_key][:] = time_array
                else:
                    root_grp[data_key][:,ii] = monthly_data[data_key]

        #write out metadata
        for metadata_key in metadata_keys:      

            #convert list of strings to character array
            #first need to convert np.object type to np.string array
            if root_grp[metadata_key].dtype == 'S1':
                char_array = stringtochar(monthly_metadata[metadata_key].astype(np.str))
                current_max_char = len(char_array)
                    
                #if characters of any of metadata exceeds character limit, limit it to max characters
                if current_max_char > root_grp[metadata_key].shape[1]:
                    char_array = char_array[:root_grp[metadata_key].shape[1]]
                    current_max_char = len(char_array)
            
                root_grp[metadata_key][ii,:current_max_char] = char_array
                
            else:
                root_grp[metadata_key][ii] = monthly_metadata[metadata_key]
                    
        #remove temporary station file
        os.remove(station_file)

    #close writing
    root_grp.close()
            
    return

###------------------------------------------------------------------------------------###

#define dictionaries which map standardised metadata name to the 3 different external metadata sources for EIONET
#syntax = 'external_metadata_file_variable_name' : 'standard_variable_name' 
def EIONET_external_meta_mapper():
    
    #for mapping variable data contained in '/esarchive/obs/ghost/EIONET/metadata/network_provided/EIONET_META.csv' to standardised names
    standard_external_meta_map = {'AirQualityStationEoICode':'station_reference',     
                                  'Latitude':                'latitude',             
                                  'Longitude':               'longitude',             
                                  'Altitude':                'altitude',
                                  'InletHeight':             'sampling_height',       
                                  'BuildingDistance':        'distance_to_building',  
                                  'KerbDistance':            'distance_to_kerb',      
                                  'AirQualityStationArea':   'area_classification',   
                                  'AirQualityStationType':   'station_classification',
                                  'SamplingPoint':           'sampling_point',        
                                  'SamplingProcess':         'sampling_process',      
                                  'Projection':              'projection',            
                                  'AirQualityNetwork':       'local_network_code',    
                                  'AirQualityStationNatCode':'local_reference',       
                                  'MeasurementEquipment':    'measurement_equipment', 
                                  'MeasurementType':         'measurement_type',
                                  'MeasurementMethod':       'measurement_method',    
                                  'AirPollutantCode':        'pollutant_code',
                                  'AirPollutant':            'pollutant'}
                                         
    #for mapping variable data contained in '/esarchive/obs/ghost/EIONET/metadata/network_provided/EIONET_AIDE_D.csv' to standardised names
    AIDE_D_external_meta_map = {'EoICode':                 'station_reference',                 
                                'Latitude':                'latitude',                          
                                'Longitude':               'longitude',                         
                                'Altitude':                'altitude',                          
                                'StationName':             'station_name',                      
                                'StationArea':             'area_classification',               
                                'StationType':             'station_classification',            
                                'ResponsibleParty':        'principal_investigator_institution',
                                'SamplingPointId':         'sampling_point',                    
                                'SamplingProcess':         'sampling_process',                  
                                'Network':                 'local_network_name',
                                'MeasurementEquipment':    'measurement_equipment',
                                'MeasurementMethod':       'measurement_method',
                                'SamplingEquipment':       'sampling_equipment',
                                'AnalyticalTechnique':     'analytical_technique',              
                                'Pollutant':               'pollutant'}
    
    #for mapping variable data contained in '/esarchive/obs/ghost/AIRBASE/metadata/processed/AIRBASE_META.npy' to standardised names
    AIRBASE_external_meta_map = {'station_european_code':                     'station_reference',                    
                                 'station_latitude_decimal_degrees':          'latitude',
                                 'station_longitude_decimal_degrees':         'longitude', 
                                 'station_altitude':                          'altitude',
                                 'height_sampling_point':                     'sampling_height',                      
                                 'station_name':                              'station_name',                         
                                 'station_type_of_area':                      'area_classification',                  
                                 'type_of_station':                           'station_classification',               
                                 'main_emission_source':                      'main_emission_source',                 
                                 'station_characteristic_of_zone':            'land_use',
                                 'station_subcategory_rural_background':      'area_classification_rural_subdivision',
                                 'station_represented_radius':                'representative_radius',                
                                 'station_city':                              'city',
                                 'station_distance_to_junction':              'distance_to_junction',                 
                                 'station_distance_to_kerb':                  'distance_to_kerb',                     
                                 'station_distance_to_source':                'distance_to_source',                   
                                 'station_street_width':                      'street_width',
                                 'street_type':                               'street_type',
                                 'number_of_vehicles':                        'daily_passing_vehicles',               
                                 'station_traffic_speed':                     'daytime_traffic_speed',                        
                                 'population':                                'population',
                                 'station_description':                       'station_description',
                                 'measurement_equipment':                     'measurement_equipment',
                                 'measurement_technique_principle':           'measurement_method',                   
                                 'measurement_automatic':                     'measurement_automatic',
                                 'calibration_method':                        'calibration_method',
                                 'calibration_frequency':                     'calibration_frequency',
                                 'length_sampling_line':                      'length_sampling_line',
                                 'location_sampling_point':                   'sample_location_miscellaneous',
                                 'EMEP_station':                              'emep_station',                         
                                 'component_code':                            'pollutant_code',                       
                                 'component_name':                            'pollutant',                            
                                 'network_code':                              'local_network_code',                   
                                 'network_short_name':                        'local_network_name',           
                                 'station_local_code':                        'local_reference',               
                                 'network_data_supplier_person_title':        'principal_investigator_title',         
                                 'network_data_supplier_person_first_name':   'principal_investigator_first_name',    
                                 'network_data_supplier_person_last_name':    'principal_investigator_last_name',     
                                 'network_data_supplier_organization_name':   'principal_investigator_institution',     
                                 'network_data_supplier_person_email_address':'principal_investigator_email_address', 
                                 'network_manager_person_title':              'contact_title',                        
                                 'network_manager_person_first_name':         'contact_first_name',                   
                                 'network_manager_person_last_name':          'contact_last_name',                    
                                 'network_manager_organization_name':         'contact_institution',                  
                                 'network_manager_person_email_address':      'contact_email_address'}                         
    
    
    return standard_external_meta_map, AIDE_D_external_meta_map, AIRBASE_external_meta_map
        
###------------------------------------------------------------------------------------###
#dictionary for ISO3166 country code to country name
def country_ISO3166_to_name(code):
    code_dict = {
	'AF': 'AFGHANISTAN',
	'AX': 'ÅLAND ISLANDS',
	'AL': 'ALBANIA',
	'DZ': 'ALGERIA',
	'AS': 'AMERICAN SAMOA',
	'AD': 'ANDORRA',
	'AO': 'ANGOLA',
	'AI': 'ANGUILLA',
	'AQ': 'ANTARCTICA',
	'AG': 'ANTIGUA AND BARBUDA',
	'AR': 'ARGENTINA',
	'AM': 'ARMENIA',
	'AW': 'ARUBA',
	'AU': 'AUSTRALIA',
	'AT': 'AUSTRIA',
	'AZ': 'AZERBAIJAN',
	'BS': 'BAHAMAS',
	'BH': 'BAHRAIN',
	'BD': 'BANGLADESH',
	'BB': 'BARBADOS',
	'BY': 'BELARUS',
	'BE': 'BELGIUM',
	'BZ': 'BELIZE',
	'BJ': 'BENIN',
	'BM': 'BERMUDA',
	'BT': 'BHUTAN',
	'BO': 'BOLIVIA, PLURINATIONAL STATE OF',
	'BQ': 'BONAIRE, SINT EUSTATIUS AND SABA',
	'BA': 'BOSNIA AND HERZEGOVINA',
	'BW': 'BOTSWANA',
	'BV': 'BOUVET ISLAND',
	'BR': 'BRAZIL',
	'IO': 'BRITISH INDIAN OCEAN TERRITORY',
	'BN': 'BRUNEI DARUSSALAM',
	'BG': 'BULGARIA',
	'BF': 'BURKINA FASO',
	'BI': 'BURUNDI',
	'KH': 'CAMBODIA',
	'CM': 'CAMEROON',
	'CA': 'CANADA',
	'CV': 'CAPE VERDE',
	'KY': 'CAYMAN ISLANDS',
	'CF': 'CENTRAL AFRICAN REPUBLIC',
	'TD': 'CHAD',
	'CL': 'CHILE',
	'CN': 'CHINA',
	'CX': 'CHRISTMAS ISLAND',
	'CC': 'COCOS (KEELING) ISLANDS',
	'CO': 'COLOMBIA',
	'KM': 'COMOROS',
	'CG': 'CONGO',
	'CD': 'CONGO, THE DEMOCRATIC REPUBLIC OF THE',
	'CK': 'COOK ISLANDS',
	'CR': 'COSTA RICA',
	'CI': 'CÔTE D\'IVOIRE',
	'HR': 'CROATIA',
	'CU': 'CUBA',
	'CW': 'CURAÇAO',
	'CY': 'CYPRUS',
	'CZ': 'CZECH REPUBLIC',
	'DK': 'DENMARK',
	'DJ': 'DJIBOUTI',
	'DM': 'DOMINICA',
	'DO': 'DOMINICAN REPUBLIC',
	'EC': 'ECUADOR',
	'EG': 'EGYPT',
	'SV': 'EL SALVADOR',
	'GQ': 'EQUATORIAL GUINEA',
	'ER': 'ERITREA',
	'EE': 'ESTONIA',
	'ET': 'ETHIOPIA',
	'FK': 'FALKLAND ISLANDS (MALVINAS)',
	'FO': 'FAROE ISLANDS',
	'FJ': 'FIJI',
	'FI': 'FINLAND',
	'FR': 'FRANCE',
	'GF': 'FRENCH GUIANA',
	'PF': 'FRENCH POLYNESIA',
	'TF': 'FRENCH SOUTHERN TERRITORIES',
	'GA': 'GABON',
	'GM': 'GAMBIA',
	'GE': 'GEORGIA',
	'DE': 'GERMANY',
	'GH': 'GHANA',
	'GI': 'GIBRALTAR',
	'GR': 'GREECE',
	'GL': 'GREENLAND',
	'GD': 'GRENADA',
	'GP': 'GUADELOUPE',
	'GU': 'GUAM',
	'GT': 'GUATEMALA',
	'GG': 'GUERNSEY',
	'GN': 'GUINEA',
	'GW': 'GUINEA-BISSAU',
	'GY': 'GUYANA',
	'HT': 'HAITI',
	'HM': 'HEARD ISLAND AND MCDONALD ISLANDS',
	'VA': 'HOLY SEE (VATICAN CITY STATE)',
	'HN': 'HONDURAS',
	'HK': 'HONG KONG',
	'HU': 'HUNGARY',
	'IS': 'ICELAND',
	'IN': 'INDIA',
	'ID': 'INDONESIA',
	'IR': 'IRAN, ISLAMIC REPUBLIC OF',
	'IQ': 'IRAQ',
	'IE': 'IRELAND',
	'IM': 'ISLE OF MAN',
	'IL': 'ISRAEL',
	'IT': 'ITALY',
	'JM': 'JAMAICA',
	'JP': 'JAPAN',
	'JE': 'JERSEY',
	'JO': 'JORDAN',
	'KZ': 'KAZAKHSTAN',
	'KE': 'KENYA',
	'KI': 'KIRIBATI',
	'KP': 'KOREA, DEMOCRATIC PEOPLE\'S REPUBLIC OF',
	'KR': 'KOREA, REPUBLIC OF',
	'KW': 'KUWAIT',
	'KG': 'KYRGYZSTAN',
	'LA': 'LAO PEOPLE\'S DEMOCRATIC REPUBLIC',
	'LV': 'LATVIA',
	'LB': 'LEBANON',
	'LS': 'LESOTHO',
	'LR': 'LIBERIA',
	'LY': 'LIBYAN ARAB JAMAHIRIYA',
	'LI': 'LIECHTENSTEIN',
	'LT': 'LITHUANIA',
	'LU': 'LUXEMBOURG',
	'MO': 'MACAO',
	'MK': 'MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF',
	'MG': 'MADAGASCAR',
	'MW': 'MALAWI',
	'MY': 'MALAYSIA',
	'MV': 'MALDIVES',
	'ML': 'MALI',
	'MT': 'MALTA',
	'MH': 'MARSHALL ISLANDS',
	'MQ': 'MARTINIQUE',
	'MR': 'MAURITANIA',
	'MU': 'MAURITIUS',
	'YT': 'MAYOTTE',
	'MX': 'MEXICO',
	'FM': 'MICRONESIA, FEDERATED STATES OF',
	'MD': 'MOLDOVA, REPUBLIC OF',
	'MC': 'MONACO',
	'MN': 'MONGOLIA',
	'ME': 'MONTENEGRO',
	'MS': 'MONTSERRAT',
	'MA': 'MOROCCO',
	'MZ': 'MOZAMBIQUE',
	'MM': 'MYANMAR',
	'NA': 'NAMIBIA',
	'NR': 'NAURU',
	'NP': 'NEPAL',
	'NL': 'NETHERLANDS',
	'NC': 'NEW CALEDONIA',
	'NZ': 'NEW ZEALAND',
	'NI': 'NICARAGUA',
	'NE': 'NIGER',
	'NG': 'NIGERIA',
	'NU': 'NIUE',
	'NF': 'NORFOLK ISLAND',
	'MP': 'NORTHERN MARIANA ISLANDS',
	'NO': 'NORWAY',
	'OM': 'OMAN',
	'PK': 'PAKISTAN',
	'PW': 'PALAU',
	'PS': 'PALESTINIAN TERRITORY, OCCUPIED',
	'PA': 'PANAMA',
	'PG': 'PAPUA NEW GUINEA',
	'PY': 'PARAGUAY',
	'PE': 'PERU',
	'PH': 'PHILIPPINES',
	'PN': 'PITCAIRN',
	'PL': 'POLAND',
	'PT': 'PORTUGAL',
	'PR': 'PUERTO RICO',
	'QA': 'QATAR',
	'RE': 'RÉUNION',
	'RO': 'ROMANIA',
	'RU': 'RUSSIAN FEDERATION',
	'RW': 'RWANDA',
	'BL': 'SAINT BARTHÉLEMY',
	'SH': 'SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA',
	'KN': 'SAINT KITTS AND NEVIS',
	'LC': 'SAINT LUCIA',
	'MF': 'SAINT MARTIN (FRENCH PART)',
	'PM': 'SAINT PIERRE AND MIQUELON',
	'VC': 'SAINT VINCENT AND THE GRENADINES',
	'WS': 'SAMOA',
	'SM': 'SAN MARINO',
	'ST': 'SAO TOME AND PRINCIPE',
	'SA': 'SAUDI ARABIA',
	'SN': 'SENEGAL',
	'RS': 'SERBIA',
	'SC': 'SEYCHELLES',
	'SL': 'SIERRA LEONE',
	'SG': 'SINGAPORE',
	'SX': 'SINT MAARTEN (DUTCH PART)',
	'SK': 'SLOVAKIA',
	'SI': 'SLOVENIA',
	'SB': 'SOLOMON ISLANDS',
	'SO': 'SOMALIA',
	'ZA': 'SOUTH AFRICA',
	'GS': 'SOUTH GEORGIA AND THE SOUTH SANDWICH ISLANDS',
	'SS': 'SOUTH SUDAN',
	'ES': 'SPAIN',
	'LK': 'SRI LANKA',
	'SD': 'SUDAN',
	'SR': 'SURINAME',
	'SJ': 'SVALBARD AND JAN MAYEN',
	'SZ': 'SWAZILAND',
	'SE': 'SWEDEN',
	'CH': 'SWITZERLAND',
	'SY': 'SYRIAN ARAB REPUBLIC',
	'TW': 'TAIWAN, PROVINCE OF CHINA',
	'TJ': 'TAJIKISTAN',
	'TZ': 'TANZANIA, UNITED REPUBLIC OF',
	'TH': 'THAILAND',
	'TL': 'TIMOR-LESTE',
	'TG': 'TOGO',
	'TK': 'TOKELAU',
	'TO': 'TONGA',
	'TT': 'TRINIDAD AND TOBAGO',
	'TN': 'TUNISIA',
	'TR': 'TURKEY',
	'TM': 'TURKMENISTAN',
	'TC': 'TURKS AND CAICOS ISLANDS',
	'TV': 'TUVALU',
	'UG': 'UGANDA',
	'UA': 'UKRAINE',
	'AE': 'UNITED ARAB EMIRATES',
	'GB': 'UNITED KINGDOM',
	'US': 'UNITED STATES',
	'UM': 'UNITED STATES MINOR OUTLYING ISLANDS',
	'UY': 'URUGUAY',
	'UZ': 'UZBEKISTAN',
	'VU': 'VANUATU',
	'VE': 'VENEZUELA, BOLIVARIAN REPUBLIC OF',
	'VN': 'VIET NAM',
	'VG': 'VIRGIN ISLANDS, BRITISH',
	'VI': 'VIRGIN ISLANDS, U.S.',
	'WF': 'WALLIS AND FUTUNA',
	'EH': 'WESTERN SAHARA',
	'XK': 'KOSOVO',
	'YE': 'YEMEN',
	'ZM': 'ZAMBIA',
	'ZW': 'ZIMBABWE'}
    return code_dict[code]
