import bisect
import copy
import datetime
import glob
import os
import re
import shutil
import sys
import time

import numpy as np
import pandas as pd
from scipy.sparse import coo_matrix
from scipy.signal import convolve, gaussian

from GHOST_standards import standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code, standard_simple_data_flag_name_to_simple_data_flag_code

def get_default_parameters_per_network(network):

    '''return default parameters to process per network'''

    parameters_d = { 
    'AERONET_v3_lev1.5':['AOD_380nm','AOD_440nm','AOD_500nm','AOD_500nm_COARSE','AOD_500nm_FINE','FINE_MODE_FRAC_500nm','AOD_675nm','AOD_870nm','AOD_1020nm','AE_440-870nm','EXTAOD_440nm','EXTAOD_440nm_COARSE','EXTAOD_440nm_FINE','EXTAOD_675nm','EXTAOD_675nm_COARSE','EXTAOD_675nm_FINE','EXTAOD_870nm','EXTAOD_870nm_COARSE','EXTAOD_870nm_FINE','EXTAOD_1020nm','EXTAOD_1020nm_COARSE','EXTAOD_1020nm_FINE','EXTAE_440-870nm','ABSAOD_440nm','ABSAOD_675nm','ABSAOD_870nm','ABSAOD_1020nm','ABSAE_440-870nm','SSA_440nm','SSA_675nm','SSA_870nm','SSA_1020nm','ASY_440nm','ASY_440nm_COARSE','ASY_440nm_FINE','ASY_675nm','ASY_675nm_COARSE','ASY_675nm_FINE','ASY_870nm','ASY_870nm_COARSE','ASY_870nm_FINE','ASY_1020nm','ASY_1020nm_COARSE','ASY_1020nm_FINE','SPH','RIN_REAL_440nm','RIN_REAL_675nm','RIN_REAL_870nm','RIN_REAL_1020nm','RIN_IMAG_440nm','RIN_IMAG_675nm','RIN_IMAG_870nm','RIN_IMAG_1020nm','VCONC','VCONC_FINE','VCONC_COARSE','VCONC_BIN1','VCONC_BIN2','VCONC_BIN3','VCONC_BIN4','VCONC_BIN5','VCONC_BIN6','VCONC_BIN7','VCONC_BIN8','VCONC_BIN9','VCONC_BIN10','VCONC_BIN11','VCONC_BIN12','VCONC_BIN13','VCONC_BIN14','VCONC_BIN15','VCONC_BIN16','VCONC_BIN17','VCONC_BIN18','VCONC_BIN19','VCONC_BIN20','VCONC_BIN21','VCONC_BIN22'],                     
    'AERONET_v3_lev2.0':['AOD_380nm','AOD_440nm','AOD_500nm','AOD_500nm_COARSE','AOD_500nm_FINE','FINE_MODE_FRAC_500nm','AOD_675nm','AOD_870nm','AOD_1020nm','AE_440-870nm','EXTAOD_440nm','EXTAOD_440nm_COARSE','EXTAOD_440nm_FINE','EXTAOD_675nm','EXTAOD_675nm_COARSE','EXTAOD_675nm_FINE','EXTAOD_870nm','EXTAOD_870nm_COARSE','EXTAOD_870nm_FINE','EXTAOD_1020nm','EXTAOD_1020nm_COARSE','EXTAOD_1020nm_FINE','EXTAE_440-870nm','ABSAOD_440nm','ABSAOD_675nm','ABSAOD_870nm','ABSAOD_1020nm','ABSAE_440-870nm','SSA_440nm','SSA_675nm','SSA_870nm','SSA_1020nm','ASY_440nm','ASY_440nm_COARSE','ASY_440nm_FINE','ASY_675nm','ASY_675nm_COARSE','ASY_675nm_FINE','ASY_870nm','ASY_870nm_COARSE','ASY_870nm_FINE','ASY_1020nm','ASY_1020nm_COARSE','ASY_1020nm_FINE','SPH','RIN_REAL_440nm','RIN_REAL_675nm','RIN_REAL_870nm','RIN_REAL_1020nm','RIN_IMAG_440nm','RIN_IMAG_675nm','RIN_IMAG_870nm','RIN_IMAG_1020nm','VCONC','VCONC_FINE','VCONC_COARSE','VCONC_BIN1','VCONC_BIN2','VCONC_BIN3','VCONC_BIN4','VCONC_BIN5','VCONC_BIN6','VCONC_BIN7','VCONC_BIN8','VCONC_BIN9','VCONC_BIN10','VCONC_BIN11','VCONC_BIN12','VCONC_BIN13','VCONC_BIN14','VCONC_BIN15','VCONC_BIN16','VCONC_BIN17','VCONC_BIN18','VCONC_BIN19','VCONC_BIN20','VCONC_BIN21','VCONC_BIN22'],                     
    'BJMEMC':['O3','NO2','SO2','CO','PM10','PM2.5'],
    'CANADA_NAPS':['O3','NO','NO2','SO2','CO','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE','NH3','HNO3','HCHO','C','Ca++','Cl-','EC','K+','Mg++','Mn','MSA','Na+','NH4+','NO3-','OC','SO4--','PM10','PM10_Al','PM10_As','PM10_C','PM10_Ca++','PM10_Cd','PM10_Cl-','PM10_Co','PM10_Cr','PM10_Cu','PM10_EC','PM10_Fe','PM10_K+','PM10_Mg++','PM10_Mn','PM10_MSA','PM10_Na+','PM10_NH4+','PM10_Ni','PM10_NO3-','PM10_OC','PM10_Pb','PM10_Se','PM10_SO4--','PM10_V','PM10_Zn','PM2.5','PM2.5_Al','PM2.5_As','PM2.5_C','PM2.5_Ca++','PM2.5_Cd','PM2.5_Cl-','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_EC','PM2.5_Fe','PM2.5_K+','PM2.5_Mg++','PM2.5_Mn','PM2.5_MSA','PM2.5_Na+','PM2.5_NH4+','PM2.5_Ni','PM2.5_NO3-','PM2.5_OC','PM2.5_Pb','PM2.5_Se','PM2.5_SO4--','PM2.5_V','PM2.5_Zn'], 
    'CAPMoN':['O3','PM10_Al','PM10_As','PM10_Cd','PM10_Co','PM10_Cr','PM10_Cu','PM10_Fe','PM10_Mn','PM10_Ni','PM10_Pb','PM10_Se','PM10_V','PM10_Zn'], 
    'CHILE_SINCA':['O3','NO','NO2','CO','SO2','CH4','NMHC','HC','PM10','PM2.5','PM10_As','PM10_Cu','PM10_Pb'], 
    'CNEMC':['O3','NO2','SO2','CO','PM10','PM2.5'],
    'EANET':['O3','NO','NO2','SO2','NH3','HNO3','HCl','PM10','PM2.5','Ca++','Cl-','K+','Mg++','Na+','NH4+','NO3-','SO4--'], 
    'EBAS':['O3','NO','NO2','SO2','CO','CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE','MONOTERPENES','VOC','NH3','HNO3','PAN','HCHO','HCl','HF','H2S','PM10','PM2.5','PM1','Al','As','BC','C','Ca++','Cd','Cl-','Co','Cr','Cu','EC','Fe','Hg','K+','Mg++','Mn','MSA','Na+','NH4+','NH4NO3','Ni','NO3-','OC','Pb','Se','SO4--','SO4--_NSS','SO4--_SS','V','Zn','PM10_Al','PM10_As','PM10_BC','PM10_C','PM10_Ca++','PM10_Cd','PM10_Cl-','PM10_Co','PM10_Cr','PM10_Cu','PM10_EC','PM10_Fe','PM10_Hg','PM10_K+','PM10_Mg++','PM10_Mn','PM10_MSA','PM10_Na+','PM10_NH4+','PM10_NH4NO3','PM10_Ni','PM10_NO3-','PM10_OC','PM10_Pb','PM10_Se','PM10_SO4--','PM10_SO4--_NSS','PM10_SO4--_SS','PM10_V','PM10_Zn','PM2.5_Al','PM2.5_As','PM2.5_BC','PM2.5_C','PM2.5_Ca++','PM2.5_Cd','PM2.5_Cl-','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_EC','PM2.5_Fe','PM2.5_Hg','PM2.5_K+','PM2.5_Mg++','PM2.5_Mn','PM2.5_MSA','PM2.5_Na+','PM2.5_NH4+','PM2.5_NH4NO3','PM2.5_Ni','PM2.5_NO3-','PM2.5_OC','PM2.5_Pb','PM2.5_Se','PM2.5_SO4--','PM2.5_SO4--_NSS','PM2.5_SO4--_SS','PM2.5_V','PM2.5_Zn','PM1_Al','PM1_As','PM1_BC','PM1_C','PM1_Ca++','PM1_Cd','PM1_Cl-','PM1_Co','PM1_Cr','PM1_Cu','PM1_EC','PM1_Fe','PM1_Hg','PM1_K+','PM1_Mg++','PM1_Mn','PM1_MSA','PM1_Na+','PM1_NH4+','PM1_NH4NO3','PM1_Ni','PM1_NO3-','PM1_OC','PM1_Pb','PM1_Se','PM1_SO4--','PM1_SO4--_NSS','PM1_SO4--_SS','PM1_V','PM1_Zn','AERO_ABS_COEFF_370nm','AERO_ABS_COEFF_467nm','AERO_ABS_COEFF_470nm','AERO_ABS_COEFF_520nm','AERO_ABS_COEFF_522nm','AERO_ABS_COEFF_525nm','AERO_ABS_COEFF_528nm','AERO_ABS_COEFF_530nm','AERO_ABS_COEFF_550nm','AERO_ABS_COEFF_565nm','AERO_ABS_COEFF_590nm','AERO_ABS_COEFF_637nm','AERO_ABS_COEFF_652nm','AERO_ABS_COEFF_660nm','AERO_ABS_COEFF_670nm','AERO_ABS_COEFF_880nm','AERO_ABS_COEFF_950nm','AERO_LIGHT_BACKSCAT_COEFF_450nm','AERO_LIGHT_BACKSCAT_COEFF_520nm','AERO_LIGHT_BACKSCAT_COEFF_525nm','AERO_LIGHT_BACKSCAT_COEFF_530nm','AERO_LIGHT_BACKSCAT_COEFF_532nm','AERO_LIGHT_BACKSCAT_COEFF_550nm','AERO_LIGHT_BACKSCAT_COEFF_635nm','AERO_LIGHT_BACKSCAT_COEFF_700nm','AERO_LIGHT_BACKSCAT_COEFF_850nm','AERO_LIGHT_SCAT_COEFF_450nm','AERO_LIGHT_SCAT_COEFF_520nm','AERO_LIGHT_SCAT_COEFF_525nm','AERO_LIGHT_SCAT_COEFF_530nm','AERO_LIGHT_SCAT_COEFF_532nm','AERO_LIGHT_SCAT_COEFF_550nm','AERO_LIGHT_SCAT_COEFF_635nm','AERO_LIGHT_SCAT_COEFF_700nm','AERO_LIGHT_SCAT_COEFF_850nm','PM10_AERO_ABS_COEFF_370nm','PM10_AERO_ABS_COEFF_467nm','PM10_AERO_ABS_COEFF_470nm','PM10_AERO_ABS_COEFF_520nm','PM10_AERO_ABS_COEFF_522nm','PM10_AERO_ABS_COEFF_525nm','PM10_AERO_ABS_COEFF_528nm','PM10_AERO_ABS_COEFF_530nm','PM10_AERO_ABS_COEFF_550nm','PM10_AERO_ABS_COEFF_565nm','PM10_AERO_ABS_COEFF_590nm','PM10_AERO_ABS_COEFF_637nm','PM10_AERO_ABS_COEFF_652nm','PM10_AERO_ABS_COEFF_660nm','PM10_AERO_ABS_COEFF_670nm','PM10_AERO_ABS_COEFF_880nm','PM10_AERO_ABS_COEFF_950nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM10_AERO_LIGHT_SCAT_COEFF_450nm','PM10_AERO_LIGHT_SCAT_COEFF_520nm','PM10_AERO_LIGHT_SCAT_COEFF_525nm','PM10_AERO_LIGHT_SCAT_COEFF_530nm','PM10_AERO_LIGHT_SCAT_COEFF_532nm','PM10_AERO_LIGHT_SCAT_COEFF_550nm','PM10_AERO_LIGHT_SCAT_COEFF_635nm','PM10_AERO_LIGHT_SCAT_COEFF_700nm','PM10_AERO_LIGHT_SCAT_COEFF_850nm','PM2.5_AERO_ABS_COEFF_370nm','PM2.5_AERO_ABS_COEFF_467nm','PM2.5_AERO_ABS_COEFF_470nm','PM2.5_AERO_ABS_COEFF_520nm','PM2.5_AERO_ABS_COEFF_522nm','PM2.5_AERO_ABS_COEFF_525nm','PM2.5_AERO_ABS_COEFF_528nm','PM2.5_AERO_ABS_COEFF_530nm','PM2.5_AERO_ABS_COEFF_550nm','PM2.5_AERO_ABS_COEFF_565nm','PM2.5_AERO_ABS_COEFF_590nm','PM2.5_AERO_ABS_COEFF_637nm','PM2.5_AERO_ABS_COEFF_652nm','PM2.5_AERO_ABS_COEFF_660nm','PM2.5_AERO_ABS_COEFF_670nm','PM2.5_AERO_ABS_COEFF_880nm','PM2.5_AERO_ABS_COEFF_950nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM2.5_AERO_LIGHT_SCAT_COEFF_450nm','PM2.5_AERO_LIGHT_SCAT_COEFF_520nm','PM2.5_AERO_LIGHT_SCAT_COEFF_525nm','PM2.5_AERO_LIGHT_SCAT_COEFF_530nm','PM2.5_AERO_LIGHT_SCAT_COEFF_532nm','PM2.5_AERO_LIGHT_SCAT_COEFF_550nm','PM2.5_AERO_LIGHT_SCAT_COEFF_635nm','PM2.5_AERO_LIGHT_SCAT_COEFF_700nm','PM2.5_AERO_LIGHT_SCAT_COEFF_850nm','PM1_AERO_ABS_COEFF_370nm','PM1_AERO_ABS_COEFF_467nm','PM1_AERO_ABS_COEFF_470nm','PM1_AERO_ABS_COEFF_520nm','PM1_AERO_ABS_COEFF_522nm','PM1_AERO_ABS_COEFF_525nm','PM1_AERO_ABS_COEFF_528nm','PM1_AERO_ABS_COEFF_530nm','PM1_AERO_ABS_COEFF_550nm','PM1_AERO_ABS_COEFF_565nm','PM1_AERO_ABS_COEFF_590nm','PM1_AERO_ABS_COEFF_637nm','PM1_AERO_ABS_COEFF_652nm','PM1_AERO_ABS_COEFF_660nm','PM1_AERO_ABS_COEFF_670nm','PM1_AERO_ABS_COEFF_880nm','PM1_AERO_ABS_COEFF_950nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM1_AERO_LIGHT_SCAT_COEFF_450nm','PM1_AERO_LIGHT_SCAT_COEFF_520nm','PM1_AERO_LIGHT_SCAT_COEFF_525nm','PM1_AERO_LIGHT_SCAT_COEFF_530nm','PM1_AERO_LIGHT_SCAT_COEFF_532nm','PM1_AERO_LIGHT_SCAT_COEFF_550nm','PM1_AERO_LIGHT_SCAT_COEFF_635nm','PM1_AERO_LIGHT_SCAT_COEFF_700nm','PM1_AERO_LIGHT_SCAT_COEFF_850nm'], 
    'EEA_AIRBASE':['O3','NO','NO2','SO2','CO','CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE','VOC','NMHC','NH3','HNO3','PAN','HCHO','HCl','HF','H2S','PM10','PM2.5','PM1','Al','As','BC','C','Ca++','Cd','Cl-','Co','Cr','Cu','EC','Fe','Hg','K+','Mg++','Mn','MSA','Na+','NH4+','NH4NO3','Ni','NO3-','OC','Pb','Se','SO4--','SO4--_NSS','SO4--_SS','V','Zn','PM10_Al','PM10_As','PM10_BC','PM10_C','PM10_Ca++','PM10_Cd','PM10_Cl-','PM10_Co','PM10_Cr','PM10_Cu','PM10_EC','PM10_Fe','PM10_Hg','PM10_K+','PM10_Mg++','PM10_Mn','PM10_MSA','PM10_Na+','PM10_NH4+','PM10_NH4NO3','PM10_Ni','PM10_NO3-','PM10_OC','PM10_Pb','PM10_Se','PM10_SO4--','PM10_SO4--_NSS','PM10_SO4--_SS','PM10_V','PM10_Zn','PM2.5_Al','PM2.5_As','PM2.5_BC','PM2.5_C','PM2.5_Ca++','PM2.5_Cd','PM2.5_Cl-','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_EC','PM2.5_Fe','PM2.5_Hg','PM2.5_K+','PM2.5_Mg++','PM2.5_Mn','PM2.5_MSA','PM2.5_Na+','PM2.5_NH4+','PM2.5_NH4NO3','PM2.5_Ni','PM2.5_NO3-','PM2.5_OC','PM2.5_Pb','PM2.5_Se','PM2.5_SO4--','PM2.5_SO4--_NSS','PM2.5_SO4--_SS','PM2.5_V','PM2.5_Zn','PM1_Al','PM1_As','PM1_BC','PM1_C','PM1_Ca++','PM1_Cd','PM1_Cl-','PM1_Co','PM1_Cr','PM1_Cu','PM1_EC','PM1_Fe','PM1_Hg','PM1_K+','PM1_Mg++','PM1_Mn','PM1_MSA','PM1_Na+','PM1_NH4+','PM1_NH4NO3','PM1_Ni','PM1_NO3-','PM1_OC','PM1_Pb','PM1_Se','PM1_SO4--','PM1_SO4--_NSS','PM1_SO4--_SS','PM1_V','PM1_Zn'],  
    'EEA_AQ_eReporting':['O3','NO','NO2','SO2','CO','CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE','VOC','NMHC','NH3','HNO3','PAN','HCHO','HCl','HF','H2S','PM10','PM2.5','PM1','Al','As','BC','C','Ca++','Cd','Cl-','Co','Cr','Cu','EC','Fe','Hg','K+','Mg++','Mn','MSA','Na+','NH4+','NH4NO3','Ni','NO3-','OC','Pb','Se','SO4--','SO4--_NSS','SO4--_SS','V','Zn','PM10_Al','PM10_As','PM10_BC','PM10_C','PM10_Ca++','PM10_Cd','PM10_Cl-','PM10_Co','PM10_Cr','PM10_Cu','PM10_EC','PM10_Fe','PM10_Hg','PM10_K+','PM10_Mg++','PM10_Mn','PM10_MSA','PM10_Na+','PM10_NH4+','PM10_NH4NO3','PM10_Ni','PM10_NO3-','PM10_OC','PM10_Pb','PM10_Se','PM10_SO4--','PM10_SO4--_NSS','PM10_SO4--_SS','PM10_V','PM10_Zn','PM2.5_Al','PM2.5_As','PM2.5_BC','PM2.5_C','PM2.5_Ca++','PM2.5_Cd','PM2.5_Cl-','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_EC','PM2.5_Fe','PM2.5_Hg','PM2.5_K+','PM2.5_Mg++','PM2.5_Mn','PM2.5_MSA','PM2.5_Na+','PM2.5_NH4+','PM2.5_NH4NO3','PM2.5_Ni','PM2.5_NO3-','PM2.5_OC','PM2.5_Pb','PM2.5_Se','PM2.5_SO4--','PM2.5_SO4--_NSS','PM2.5_SO4--_SS','PM2.5_V','PM2.5_Zn','PM1_Al','PM1_As','PM1_BC','PM1_C','PM1_Ca++','PM1_Cd','PM1_Cl-','PM1_Co','PM1_Cr','PM1_Cu','PM1_EC','PM1_Fe','PM1_Hg','PM1_K+','PM1_Mg++','PM1_Mn','PM1_MSA','PM1_Na+','PM1_NH4+','PM1_NH4NO3','PM1_Ni','PM1_NO3-','PM1_OC','PM1_Pb','PM1_Se','PM1_SO4--','PM1_SO4--_NSS','PM1_SO4--_SS','PM1_V','PM1_Zn'], 
    'JAPAN_NIES':['O3','NO','NO2','CO','SO2','CH4','NMHC','HC','HCl','HF','H2S','PM10','PM2.5'], 
    'MEXICO_CDMX':['O3','NO','NO2','CO','SO2','PM10','PM2.5'], 
    'MITECO':['O3','NO','NO2','CO','SO2','BENZENE','TOLUENE','PM10','PM2.5','PM10_As','PM10_Cd','PM10_Ni','PM10_Pb'], 
    'NOAA_ISD':['WND_DIR_10M','WND_SPD_10M','TEMP_2M','RH_2M','SST','DEW_PT_2M','PRES_2M','SLP','PREC_ACCUM','SNOW_ACCUM','SNOW_DEPTH','CEILING_HEIGHT','VIS_DIST','CLOUD_CVG_FRAC'], 
    'SEARCH':[], 
    'UK_AIR':['O3','NO','NO2','SO2','CO','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE','NH3','HNO3','HCl','Al','As','BC','Cd','Co','Cr','Cu','Fe','Mn','Ni','Pb','Se','V','Zn','PM10','PM10_C','PM10_Ca++','PM10_Cl-','PM10_EC','PM10_K+','PM10_Mg++','PM10_Na+','PM10_NH4+','PM10_NO3-','PM10_OC','PM10_SO4--','PM2.5','PM2.5_C','PM2.5_Ca++','PM2.5_Cl-','PM2.5_EC','PM2.5_Hg','PM2.5_K+','PM2.5_Mg++','PM2.5_Na+','PM2.5_NH4+','PM2.5_NO3-','PM2.5_OC','PM2.5_SO4--','PM1_NH4+','PM1_NO3-','PM1_OC','PM1_SO4--'], 
    'US_EPA_AirNow_DOS':['O3','PM10','PM2.5'],
    'US_EPA_AQS':['O3','NO','NO2','SO2','CO','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE','NMVOC','HCHO','Pb','PM10','PM10_Al','PM10_As','PM10_C','PM10_Ca++','PM10_Cd','PM10_Cl-','PM10_Co','PM10_Cr','PM10_Cu','PM10_EC','PM10_Fe','PM10_Hg','PM10_K+','PM10_Mg++','PM10_Mn','PM10_Na+','PM10_NH4+','PM10_Ni','PM10_NO3-','PM10_OC','PM10_Pb','PM10_Se','PM10_SO4--','PM10_V','PM10_Zn','PM2.5','PM2.5_Al','PM2.5_As','PM2.5_C','PM2.5_Ca++','PM2.5_Cd','PM2.5_Cl-','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_EC','PM2.5_Fe','PM2.5_Hg','PM2.5_K+','PM2.5_Mg++','PM2.5_Mn','PM2.5_Na+','PM2.5_NH4+','PM2.5_Ni','PM2.5_NO3-','PM2.5_OC','PM2.5_Pb','PM2.5_Se','PM2.5_SO4--','PM2.5_V','PM2.5_Zn'], 
    'US_EPA_CASTNET':['O3','NO','NO2','CO','NH3','SO2','HNO3','Ca++','Cl-','K+','Mg++','Na+','NH4+','NO3-','SO4--','PM2.5','PM2.5_Al','PM2.5_As','PM2.5_C','PM2.5_Ca++','PM2.5_Cd','PM2.5_Cl-','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_EC','PM2.5_Fe','PM2.5_Hg','PM2.5_K+','PM2.5_Mg++','PM2.5_Mn','PM2.5_Na+','PM2.5_Ni','PM2.5_NO3-','PM2.5_OC','PM2.5_Pb','PM2.5_Se','PM2.5_SO4--','PM2.5_V','PM2.5_Zn'], 
    'US_NADP_AMNet':['PM2.5_Hg'],
    'US_NADP_AMoN':['NH3'], 
    'WMO_WDCGG':['CO','CH4']
    }

    return parameters_d[network]

def get_default_product_parameters_per_network(network):

    '''return default products parameters to calculate per network,
       each associated with the parameters neccessary for their calculation, and the function name to calculate them
    '''
    parameters_d = {
    'AERONET_v3_lev1.5':{'AOD_550nm':{'function':'calculate_aod', 'components':['AOD_440nm','AOD_675nm','AOD_870nm']}, 'EXTAOD_550nm':{'function':'calculate_aod', 'components':['EXTAOD_440nm','EXTAOD_675nm','EXTAOD_870nm']}, 'ABSAOD_550nm':{'function':'calculate_aod', 'components':['ABSAOD_440nm','ABSAOD_675nm','ABSAOD_870nm']}, 'SSA_550nm':{'function':'calculate_ssa', 'components':['EXTAOD_550nm','ABSAOD_550nm']}, 'EXTAE_440-675nm':{'function':'calculate_ae', 'components':['EXTAOD_440nm','EXTAOD_675nm']}, 'ABSAE_440-675nm':{'function':'calculate_ae', 'components':['ABSAOD_440nm','ABSAOD_675nm']}, 'ABSAE_675-870nm':{'function':'calculate_ae', 'components':['ABSAOD_675nm','ABSAOD_870nm']}, 'SSAAE_440-675nm':{'function':'calculate_ae', 'components':['SSA_440nm','SSA_675nm']}, 'SCATAE_440-675nm':{'function':'calculate_ae_alt', 'components':['EXTAE_440-675nm','SSAAE_440-675nm']}, 'ABSWDAE':{'function':'calculate_wdae', 'components':['ABSAE_440-870nm','ABSAE_675-870nm']}, 'BC_ABSAE_440-870nm':{'function':'calculate_ae_bc', 'components':['ABSAE_675-870nm']}, 'BC_ABSAOD_440nm':{'function':'calculate_aod_bc', 'components':['ABSAOD_870nm','SCATAE_440-675nm','ABSAE_440-675nm','ABSAE_440-870nm']}, 'BrC_ABSAOD_440nm':{'function':'calculate_aod_brc', 'components':['ABSAOD_440nm','BC_ABSAOD_440nm']}},
    'AERONET_v3_lev2.0':{'AOD_550nm':{'function':'calculate_aod', 'components':['AOD_440nm','AOD_675nm','AOD_870nm']}, 'EXTAOD_550nm':{'function':'calculate_aod', 'components':['EXTAOD_440nm','EXTAOD_675nm','EXTAOD_870nm']}, 'ABSAOD_550nm':{'function':'calculate_aod', 'components':['ABSAOD_440nm','ABSAOD_675nm','ABSAOD_870nm']}, 'SSA_550nm':{'function':'calculate_ssa', 'components':['EXTAOD_550nm','ABSAOD_550nm']}, 'EXTAE_440-675nm':{'function':'calculate_ae', 'components':['EXTAOD_440nm','EXTAOD_675nm']}, 'ABSAE_440-675nm':{'function':'calculate_ae', 'components':['ABSAOD_440nm','ABSAOD_675nm']}, 'ABSAE_675-870nm':{'function':'calculate_ae', 'components':['ABSAOD_675nm','ABSAOD_870nm']}, 'SSAAE_440-675nm':{'function':'calculate_ae', 'components':['SSA_440nm','SSA_675nm']}, 'SCATAE_440-675nm':{'function':'calculate_ae_alt', 'components':['EXTAE_440-675nm','SSAAE_440-675nm']}, 'ABSWDAE':{'function':'calculate_wdae', 'components':['ABSAE_440-870nm','ABSAE_675-870nm']}, 'BC_ABSAE_440-870nm':{'function':'calculate_ae_bc', 'components':['ABSAE_675-870nm']}, 'BC_ABSAOD_440nm':{'function':'calculate_aod_bc', 'components':['ABSAOD_870nm','SCATAE_440-675nm','ABSAE_440-675nm','ABSAE_440-870nm']}, 'BrC_ABSAOD_440nm':{'function':'calculate_aod_brc', 'components':['ABSAOD_440nm','BC_ABSAOD_440nm']}},
    'EBAS':             {'ABSAE_370-880nm':{'function':'calculate_ae', 'components':['AERO_ABS_COEFF_370nm','AERO_ABS_COEFF_880nm']}, 'ABSAE_370-950nm':{'function':'calculate_ae', 'components':['AERO_ABS_COEFF_370nm','AERO_ABS_COEFF_470nm','AERO_ABS_COEFF_520nm','AERO_ABS_COEFF_590nm','AERO_ABS_COEFF_660nm','AERO_ABS_COEFF_880nm','AERO_ABS_COEFF_950nm']}, 'ABSAE_467-652nm':{'function':'calculate_ae', 'components':['AERO_ABS_COEFF_467nm','AERO_ABS_COEFF_528nm','AERO_ABS_COEFF_652nm']}, 'ABSAE_467-660nm':{'function':'calculate_ae', 'components':['AERO_ABS_COEFF_467nm','AERO_ABS_COEFF_530nm','AERO_ABS_COEFF_550nm','AERO_ABS_COEFF_660nm'], 'option':['AERO_ABS_COEFF_530nm','AERO_ABS_COEFF_550nm']}, 'SCATAE_450-635nm':{'function':'calculate_ae', 'components':['AERO_LIGHT_SCAT_COEFF_450nm','AERO_LIGHT_SCAT_COEFF_525nm','AERO_LIGHT_SCAT_COEFF_635nm']}, 'SCATAE_450-700nm':{'function':'calculate_ae', 'components':['AERO_LIGHT_SCAT_COEFF_450nm','AERO_LIGHT_SCAT_COEFF_550nm','AERO_LIGHT_SCAT_COEFF_700nm']}, 'BACKSCAT_FRAC_450nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_450nm','AERO_LIGHT_SCAT_COEFF_450nm']}, 'BACKSCAT_FRAC_520nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_520nm','AERO_LIGHT_SCAT_COEFF_520nm']}, 'BACKSCAT_FRAC_525nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_525nm','AERO_LIGHT_SCAT_COEFF_525nm']}, 'BACKSCAT_FRAC_530nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_530nm','AERO_LIGHT_SCAT_COEFF_530nm']}, 'BACKSCAT_FRAC_532nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_532nm','AERO_LIGHT_SCAT_COEFF_532nm']}, 'BACKSCAT_FRAC_550nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_550nm','AERO_LIGHT_SCAT_COEFF_550nm']}, 'BACKSCAT_FRAC_635nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_635nm','AERO_LIGHT_SCAT_COEFF_635nm']}, 'BACKSCAT_FRAC_700nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_700nm','AERO_LIGHT_SCAT_COEFF_700nm']}, 'BACKSCAT_FRAC_850nm':{'function':'calculate_bf', 'components':['AERO_LIGHT_BACKSCAT_COEFF_850nm','AERO_LIGHT_SCAT_COEFF_850nm']}, 'ASY_525nm':{'function':'calculate_asy', 'components':['BACKSCAT_FRAC_525nm']}, 'ASY_550nm':{'function':'calculate_asy', 'components':['BACKSCAT_FRAC_550nm']},
                         'PM10_ABSAE_370-880nm':{'function':'calculate_ae', 'components':['PM10_AERO_ABS_COEFF_370nm','PM10_AERO_ABS_COEFF_880nm']}, 'PM10_ABSAE_370-950nm':{'function':'calculate_ae', 'components':['PM10_AERO_ABS_COEFF_370nm','PM10_AERO_ABS_COEFF_470nm','PM10_AERO_ABS_COEFF_520nm','PM10_AERO_ABS_COEFF_590nm','PM10_AERO_ABS_COEFF_660nm','PM10_AERO_ABS_COEFF_880nm','PM10_AERO_ABS_COEFF_950nm']}, 'PM10_ABSAE_467-652nm':{'function':'calculate_ae', 'components':['PM10_AERO_ABS_COEFF_467nm','PM10_AERO_ABS_COEFF_528nm','PM10_AERO_ABS_COEFF_652nm']}, 'PM10_ABSAE_467-660nm':{'function':'calculate_ae', 'components':['PM10_AERO_ABS_COEFF_467nm','PM10_AERO_ABS_COEFF_530nm','PM10_AERO_ABS_COEFF_550nm','PM10_AERO_ABS_COEFF_660nm'], 'option':['PM10_AERO_ABS_COEFF_530nm','PM10_AERO_ABS_COEFF_550nm']}, 'PM10_SCATAE_450-635nm':{'function':'calculate_ae', 'components':['PM10_AERO_LIGHT_SCAT_COEFF_450nm','PM10_AERO_LIGHT_SCAT_COEFF_525nm','PM10_AERO_LIGHT_SCAT_COEFF_635nm']}, 'PM10_SCATAE_450-700nm':{'function':'calculate_ae', 'components':['PM10_AERO_LIGHT_SCAT_COEFF_450nm','PM10_AERO_LIGHT_SCAT_COEFF_550nm','PM10_AERO_LIGHT_SCAT_COEFF_700nm']}, 'PM10_BACKSCAT_FRAC_450nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM10_AERO_LIGHT_SCAT_COEFF_450nm']}, 'PM10_BACKSCAT_FRAC_520nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM10_AERO_LIGHT_SCAT_COEFF_520nm']}, 'PM10_BACKSCAT_FRAC_525nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM10_AERO_LIGHT_SCAT_COEFF_525nm']}, 'PM10_BACKSCAT_FRAC_530nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM10_AERO_LIGHT_SCAT_COEFF_530nm']}, 'PM10_BACKSCAT_FRAC_532nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM10_AERO_LIGHT_SCAT_COEFF_532nm']}, 'PM10_BACKSCAT_FRAC_550nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM10_AERO_LIGHT_SCAT_COEFF_550nm']}, 'PM10_BACKSCAT_FRAC_635nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM10_AERO_LIGHT_SCAT_COEFF_635nm']}, 'PM10_BACKSCAT_FRAC_700nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM10_AERO_LIGHT_SCAT_COEFF_700nm']}, 'PM10_BACKSCAT_FRAC_850nm':{'function':'calculate_bf', 'components':['PM10_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM10_AERO_LIGHT_SCAT_COEFF_850nm']}, 'PM10_ASY_525nm':{'function':'calculate_asy', 'components':['PM10_BACKSCAT_FRAC_525nm']}, 'PM10_ASY_550nm':{'function':'calculate_asy', 'components':['PM10_BACKSCAT_FRAC_550nm']},
                         'PM2.5_ABSAE_370-880nm':{'function':'calculate_ae', 'components':['PM2.5_AERO_ABS_COEFF_370nm','PM2.5_AERO_ABS_COEFF_880nm']}, 'PM2.5_ABSAE_370-950nm':{'function':'calculate_ae', 'components':['PM2.5_AERO_ABS_COEFF_370nm','PM2.5_AERO_ABS_COEFF_470nm','PM2.5_AERO_ABS_COEFF_520nm','PM2.5_AERO_ABS_COEFF_590nm','PM2.5_AERO_ABS_COEFF_660nm','PM2.5_AERO_ABS_COEFF_880nm','PM2.5_AERO_ABS_COEFF_950nm']}, 'PM2.5_ABSAE_467-652nm':{'function':'calculate_ae', 'components':['PM2.5_AERO_ABS_COEFF_467nm','PM2.5_AERO_ABS_COEFF_528nm','PM2.5_AERO_ABS_COEFF_652nm']}, 'PM2.5_ABSAE_467-660nm':{'function':'calculate_ae', 'components':['PM2.5_AERO_ABS_COEFF_467nm','PM2.5_AERO_ABS_COEFF_530nm','PM2.5_AERO_ABS_COEFF_550nm','PM2.5_AERO_ABS_COEFF_660nm'], 'option':['PM2.5_AERO_ABS_COEFF_530nm','PM2.5_AERO_ABS_COEFF_550nm']}, 'PM2.5_SCATAE_450-635nm':{'function':'calculate_ae', 'components':['PM2.5_AERO_LIGHT_SCAT_COEFF_450nm','PM2.5_AERO_LIGHT_SCAT_COEFF_525nm','PM2.5_AERO_LIGHT_SCAT_COEFF_635nm']}, 'PM2.5_SCATAE_450-700nm':{'function':'calculate_ae', 'components':['PM2.5_AERO_LIGHT_SCAT_COEFF_450nm','PM2.5_AERO_LIGHT_SCAT_COEFF_550nm','PM2.5_AERO_LIGHT_SCAT_COEFF_700nm']}, 'PM2.5_BACKSCAT_FRAC_450nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM2.5_AERO_LIGHT_SCAT_COEFF_450nm']}, 'PM2.5_BACKSCAT_FRAC_520nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM2.5_AERO_LIGHT_SCAT_COEFF_520nm']}, 'PM2.5_BACKSCAT_FRAC_525nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM2.5_AERO_LIGHT_SCAT_COEFF_525nm']}, 'PM2.5_BACKSCAT_FRAC_530nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM2.5_AERO_LIGHT_SCAT_COEFF_530nm']}, 'PM2.5_BACKSCAT_FRAC_532nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM2.5_AERO_LIGHT_SCAT_COEFF_532nm']}, 'PM2.5_BACKSCAT_FRAC_550nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM2.5_AERO_LIGHT_SCAT_COEFF_550nm']}, 'PM2.5_BACKSCAT_FRAC_635nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM2.5_AERO_LIGHT_SCAT_COEFF_635nm']}, 'PM2.5_BACKSCAT_FRAC_700nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM2.5_AERO_LIGHT_SCAT_COEFF_700nm']}, 'PM2.5_BACKSCAT_FRAC_850nm':{'function':'calculate_bf', 'components':['PM2.5_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM2.5_AERO_LIGHT_SCAT_COEFF_850nm']}, 'PM2.5_ASY_525nm':{'function':'calculate_asy', 'components':['PM2.5_BACKSCAT_FRAC_525nm']}, 'PM2.5_ASY_550nm':{'function':'calculate_asy', 'components':['PM2.5_BACKSCAT_FRAC_550nm']},
                         'PM1_ABSAE_370-880nm':{'function':'calculate_ae', 'components':['PM1_AERO_ABS_COEFF_370nm','PM1_AERO_ABS_COEFF_880nm']}, 'PM1_ABSAE_370-950nm':{'function':'calculate_ae', 'components':['PM1_AERO_ABS_COEFF_370nm','PM1_AERO_ABS_COEFF_470nm','PM1_AERO_ABS_COEFF_520nm','PM1_AERO_ABS_COEFF_590nm','PM1_AERO_ABS_COEFF_660nm','PM1_AERO_ABS_COEFF_880nm','PM1_AERO_ABS_COEFF_950nm']}, 'PM1_ABSAE_467-652nm':{'function':'calculate_ae', 'components':['PM1_AERO_ABS_COEFF_467nm','PM1_AERO_ABS_COEFF_528nm','PM1_AERO_ABS_COEFF_652nm']}, 'PM1_ABSAE_467-660nm':{'function':'calculate_ae', 'components':['PM1_AERO_ABS_COEFF_467nm','PM1_AERO_ABS_COEFF_530nm','PM1_AERO_ABS_COEFF_550nm','PM1_AERO_ABS_COEFF_660nm'], 'option':['PM1_AERO_ABS_COEFF_530nm','PM1_AERO_ABS_COEFF_550nm']}, 'PM1_SCATAE_450-635nm':{'function':'calculate_ae', 'components':['PM1_AERO_LIGHT_SCAT_COEFF_450nm','PM1_AERO_LIGHT_SCAT_COEFF_525nm','PM1_AERO_LIGHT_SCAT_COEFF_635nm']}, 'PM1_SCATAE_450-700nm':{'function':'calculate_ae', 'components':['PM1_AERO_LIGHT_SCAT_COEFF_450nm','PM1_AERO_LIGHT_SCAT_COEFF_550nm','PM1_AERO_LIGHT_SCAT_COEFF_700nm']}, 'PM1_BACKSCAT_FRAC_450nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM1_AERO_LIGHT_SCAT_COEFF_450nm']}, 'PM1_BACKSCAT_FRAC_520nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM1_AERO_LIGHT_SCAT_COEFF_520nm']}, 'PM1_BACKSCAT_FRAC_525nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM1_AERO_LIGHT_SCAT_COEFF_525nm']}, 'PM1_BACKSCAT_FRAC_530nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM1_AERO_LIGHT_SCAT_COEFF_530nm']}, 'PM1_BACKSCAT_FRAC_532nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM1_AERO_LIGHT_SCAT_COEFF_532nm']}, 'PM1_BACKSCAT_FRAC_550nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM1_AERO_LIGHT_SCAT_COEFF_550nm']}, 'PM1_BACKSCAT_FRAC_635nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM1_AERO_LIGHT_SCAT_COEFF_635nm']}, 'PM1_BACKSCAT_FRAC_700nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM1_AERO_LIGHT_SCAT_COEFF_700nm']}, 'PM1_BACKSCAT_FRAC_850nm':{'function':'calculate_bf', 'components':['PM1_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM1_AERO_LIGHT_SCAT_COEFF_850nm']}, 'PM1_ASY_525nm':{'function':'calculate_asy', 'components':['PM1_BACKSCAT_FRAC_525nm']}, 'PM1_ASY_550nm':{'function':'calculate_asy', 'components':['PM1_BACKSCAT_FRAC_550nm']}}
    }

    return parameters_d[network]

def networks_to_merge(merge_type):

    '''return networks to be merged for a specific merge type'''

    merge_type_networks = {   
    'GHOST':       ['AERONET_v3_lev1.5','AERONET_v3_lev2.0','BJMEMC','CANADA_NAPS','CAPMoN','CHILE_SINCA','CNEMC',
                    'EANET','EBAS-ACTRIS','EBAS-AMAP','EBAS-CAMP','EBAS-COLOSSAL','EBAS-EMEP','EBAS-EUCAARI',
                    'EBAS-EUSAAR','EBAS-GUAN','EBAS-HELCOM','EBAS-HTAP','EBAS-IMPACTS','EBAS-IMPROVE',
                    'EBAS-Independent','EBAS-NILU','EBAS-NOAA_ESRL','EBAS-NOAA_GGGRN','EBAS-OECD','EBAS-UK_DECC',
                    'EBAS-WMO_WDCA','EBAS-WMO_WDCRG','EEA_AIRBASE','EEA_AQ_eReporting','JAPAN_NIES','MEXICO_CDMX',
                    'MITECO','NOAA_ISD','NOAA_ISD_EU','NOAA_ISD_NA','NOAA_ISD_IP','UK_AIR','US_EPA_AirNow_DOS',
                    'US_EPA_AQS','US_EPA_CASTNET','US_NADP_AMNet','US_NADP_AMoN','WMO_WDCGG'],
                    
    'GHOST-PUBLIC':['AERONET_v3_lev1.5','AERONET_v3_lev2.0','CANADA_NAPS','CAPMoN','CHILE_SINCA','EBAS-ACTRIS',
                    'EBAS-AMAP','EBAS-CAMP','EBAS-COLOSSAL','EBAS-EMEP','EBAS-EUCAARI','EBAS-EUSAAR','EBAS-GUAN',
                    'EBAS-HELCOM','EBAS-HTAP','EBAS-IMPACTS','EBAS-IMPROVE','EBAS-Independent','EBAS-NILU',
                    'EBAS-NOAA_ESRL','EBAS-NOAA_GGGRN','EBAS-OECD','EBAS-UK_DECC','EBAS-WMO_WDCA','EBAS-WMO_WDCRG',
                    'EEA_AIRBASE','EEA_AQ_eReporting','MEXICO_CDMX','MITECO','UK_AIR','US_EPA_AirNow_DOS',
                    'US_EPA_AQS','US_EPA_CASTNET','US_NADP_AMNet','US_NADP_AMoN','WMO_WDCGG']
    }

    return merge_type_networks[merge_type]


def get_default_temporal_resolutions_per_matrix(matrix):

    '''return default temporal resolutions to output for data matrix'''

    if matrix in ['meteo','aod','extaod','absaod','absaod-bc','absaod-brc','scataod','ssa','asy','rin','vconc','size']:
        default_temporal_resolutions = ['H','D','M','HI']
    else:
        default_temporal_resolutions = ['H','D','M']

    return default_temporal_resolutions


def get_configuration_variables(slurm_job_ID, network, parameter):

    '''return relevant specific dictionaries for specific network/parameter configuration'''

    from GHOST_standards import standard_parameters, standard_networks
    import importlib

    #Read configuration file (as was originally set when pipeline was launched)
    cwd = os.getcwd()
    if '/run' in cwd:
        sys.path.append(cwd.split('/run')[0]+'/run/GHOST_configurations')
    elif '/processing_modules' in cwd:
        sys.path.append(cwd.split('/processing_modules')[0]+'/run/GHOST_configurations')
    GHOST_configuration = importlib.import_module('configuration_{}'.format(slurm_job_ID))

    #get run details
    start_year =  GHOST_configuration.start_date[:4]
    end_year =    GHOST_configuration.end_date[:4]
    start_month = GHOST_configuration.start_date[4:]
    end_month =   GHOST_configuration.end_date[4:]
    run_details = {k: v for k, v in vars(GHOST_configuration).items() if not k.startswith('_')}
    run_details['slurm_job_ID'] = slurm_job_ID
    run_details['network'] = network
    run_details['parameter'] = parameter
    run_details['start_year'] = start_year
    run_details['end_year'] = end_year
    run_details['start_month'] = start_month
    run_details['end_month'] = end_month
    
    #get standard parameter details
    parameter_details = standard_parameters[parameter]
        
    #get standard network details
    network_details = standard_networks[network]

    return run_details, parameter_details, network_details


def create_GHOST_stage_logfile(process_code, run_details, previous_stage_data_files=[]):
    
    '''function which creates a logfile for stating outcome of a GHOST stage.
       the filename is prefixed with a code referencing the stage outcome + param_to_parallelise_by: process_code_param_to_parallelise_by.out
       the process codes are:
       0: Process completed without issue
       1: Process completed, but station no longer continuing in pipeline
       2: Caught error in process
       3: Uncaught error in process

       remove previous stage temporary data files associated with param_to_parallelise_by if configured to, if not stage S1/T1, and process code is 0 or 1
    '''
    f=open('{}/run/stage_logs/{}_{}/{}/{}_{}.out'.format(run_details['process_root'],run_details['network'],run_details['parameter'],run_details['GHOST_stage'],process_code,run_details['log_file_ref']),"w")
    f.write(str(run_details['log_file_str']))
    f.close() 

    #delete previous stage temporary data files associated with param_to_parallelise_by if configured to, if not stage S1/T1/P1/M1/E1, and process code is 0 or 1
    if run_details['remove_temporary_data_files_after_stage']:
        if run_details['GHOST_stage'] not in ['S1','T1','P1','M1','E1']:
            if int(process_code) in [0,1]:
                if type(previous_stage_data_files) == list: 
                    for data_file in previous_stage_data_files:
                        os.remove(data_file)
                else:
                    os.remove(previous_stage_data_files)

    #exit from current process after writing logfile
    sys.exit()
    

def check_directory_existence(directory_tree_str, directories_not_to_test=None):

    '''iterate through provided directory tree string
       first, check if directory exists, if not create it.
       second, check if each directory has 770 permissions.
       finally, also ensure group owner is bsc32.
    '''

    #define special characters that will need to be escaped for execution of commands      
    special_characters = ['(',')']

    #first check if instructed to not to check existence of part of directory tree str 
    if directories_not_to_test is not None:
        #modify directory_tree_str to not include directories_not_to_test
        directory_tree_str = directory_tree_str.replace(directories_not_to_test,'')
        #set directory_str_to_test to be directories_not_to_test
        directory_str_to_test = copy.deepcopy(directories_not_to_test)
    else:
        #else, set directory_str_to_test as initially empty string
        directory_str_to_test = ''

    #split directory tree str
    directory_tree_str_split = directory_tree_str.split('/')

    #iterate through directory_tree_str_split
    for current_directory in directory_tree_str_split:

        #if current directory is empty string then do not check directory existence
        if current_directory != '':

            #add current_dictionary to directory_str_to_test
            directory_str_to_test = directory_str_to_test + '/' + current_directory

            #does directory_str_to_test exist? 
            if not os.path.isdir(directory_str_to_test):
                
                #escape certain special characters in directory_str_to_test
                alt_directory_str_to_test = copy.deepcopy(directory_str_to_test)
                for ch in special_characters:
                    alt_directory_str_to_test = alt_directory_str_to_test.replace(ch,'\{}'.format(ch))

                #if not, create it
                os.system("mkdir {}".format(alt_directory_str_to_test))
                #give 770 permissions to directory
                os.system("chmod 770 {}".format(alt_directory_str_to_test))
                #make group owner Earth
                os.system("chgrp Earth {}".format(alt_directory_str_to_test))


def set_file_permissions_ownership(file_str):
    '''set 770 permissions for a newly written file and also ensure group owner is bsc32'''

    #define special characters that will need to be escaped for execution of commands      
    special_characters = ['(',')']

    #escape certain special characters in file_str
    for ch in special_characters:
        file_str = file_str.replace(ch,'\{}'.format(ch))

    #give 770 permissions to file
    os.system("chmod 770 {}".format(file_str))

    #make group owner Earth
    os.system("chgrp Earth {}".format(file_str))
        

def check_GHOST_output_directories(run_details, parameter_details, stage_type='S'):

    '''test for presence of native / output resolution directories for defined stage type, for a specific network/parameter.
       if directories do not exist, create them.
    '''

    from GHOST_standards import standard_temporal_resolutions

    #S stage type --> then create native directory 
    if stage_type == 'S':
        
        #set native directory for specific network/parameter
        native_dir = run_details['data_root'] + '{}/{}/native/{}/'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'])
    
        #check if need to create any directories in path 
        check_directory_existence(native_dir, directories_not_to_test=run_details['data_root'])

    #T stage type --> then create output resolution directories
    elif stage_type == 'T':
        #iterate through all defined temporal resolutions to process
        if run_details['temporal_resolutions'] == 'default':
            temporal_resolutions = get_default_temporal_resolutions_per_matrix(parameter_details['matrix'])
        else:
            temporal_resolutions = run_details['temporal_resolutions']
        for temporal_resolution_code in temporal_resolutions:

            #get temporal resolution specific variables
            temporal_resolution_vars = standard_temporal_resolutions[temporal_resolution_code]

            #iterate through all defined regions to process
            for region in run_details['regions']:

                #set network directory per region
                if region == 'GLOBAL':
                    network_dir = run_details['network']
                else:
                    network_dir = '{}_{}'.format(run_details['network'], region)

                #set output resolution directory for specific network/output resolution/parameter
                output_resolution_dir = run_details['data_root'] + '{}/{}/{}/{}/'.format(network_dir,run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],parameter_details['bsc_parameter_name'])
        
                #check if need to create any directories in path 
                check_directory_existence(output_resolution_dir, directories_not_to_test=run_details['data_root'])
    

def check_GHOST_stage_temporary_directories(run_details, parameter_details, stage='S1', remove_lock=False):

    '''test for presence of a specific stages'/all temporary data file directories for a specific network/parameter,
       used to store .npz station files between each of the S stages,
       and year/month chunked station files between the T stages. 
       if directories do not exist, create them.
       if directories do exist, delete all current temporary data files in directories
    '''

    from GHOST_standards import standard_temporal_resolutions

    S_GHOST_stages = []
    #if 'all' are defined to be checked get all defined S stages to process
    if stage == 'all':
        S_GHOST_stages = sorted([stage for stage in run_details['stages_to_process'] if 'S' in stage])
        #should not have a temporary directory for S3/S4
        if 'S3' in S_GHOST_stages:
            S_GHOST_stages.remove('S3')
            S3_dir = run_details['data_root']+'{}/{}/native/{}/temporary_S3/'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'])
            if os.path.isdir(S3_dir):
                shutil.rmtree(S3_dir) 
        if 'S4' in S_GHOST_stages:
            S_GHOST_stages.remove('S4')
            S4_dir = run_details['data_root']+'{}/{}/native/{}/temporary_S4/'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'])
            if os.path.isdir(S4_dir):
                shutil.rmtree(S4_dir) 

    #elif specific S stage to be checked?
    elif 'S' in stage:
        S_GHOST_stages = [stage]      

    #iterate through all defined S GHOST stage directories to check 
    for S_stage in S_GHOST_stages:
        #define temporary data directory for GHOST stage    
        temp_dir = run_details['data_root']+'{}/{}/native/{}/temporary_{}/'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],S_stage)
        
        #check if need to create any directories in path 
        check_directory_existence(temp_dir, directories_not_to_test='{}{}/{}/native/{}'.format(run_details['data_root'],run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name']))
        
        #remove all files currently in the directory

        for zippath in glob.iglob(os.path.join(temp_dir, '*.lock')):
            os.remove(zippath)  
        if not remove_lock:
            for zippath in glob.iglob(os.path.join(temp_dir, '*.npz')):
                os.remove(zippath)   
   

    #check if 'T1'/'all' is defined
    if stage in ['T1','all']:
        #if so, check year/month temporary directories for all defined temporal resolutions to output
        
        #iterate through defined temporal_resolutions to output
        if run_details['temporal_resolutions'] == 'default':
            temporal_resolutions = get_default_temporal_resolutions_per_matrix(parameter_details['matrix'])
        else:
            temporal_resolutions = run_details['temporal_resolutions']
        for temporal_resolution_code in temporal_resolutions:

            #iterate through defined regions to process
            for region in run_details['regions']:

                #set network directory per region
                if region == 'GLOBAL':
                    network_dir = run_details['network']
                else:
                    network_dir = '{}_{}'.format(run_details['network'], region)

                #get all years to process
                process_years = range(int(run_details['start_year']),int(run_details['end_year'])+1)
                for year in process_years:

                    #is it first process year?
                    if year == process_years[0]:
                        months = np.arange(int(run_details['start_month']), 13)
    
                    #is it last process year?
                    elif year == process_years[-1]:
                        months = np.arange(1,int(run_details['end_month']))
    
                    #full year of months otherwise
                    else:
                        months = np.arange(1,13)
    
                    for month in months:
                        if month < 10:
                            month_string = '0'+str(month)
                        else:
                            month_string = str(month)
                        #define temporary data directory for T1 stage  
                        temp_dir = run_details['data_root']+'{}/{}/{}/{}/temporary/{}/{}/'.format(network_dir,run_details['ghost_version'],standard_temporal_resolutions[temporal_resolution_code]['temporal_resolution_path'],parameter_details['bsc_parameter_name'],year,month_string)

                        #check if need to create any directories in path 
                        check_directory_existence(temp_dir, directories_not_to_test='{}{}/{}/{}/{}'.format(run_details['data_root'],network_dir,run_details['ghost_version'],standard_temporal_resolutions[temporal_resolution_code]['temporal_resolution_path'],parameter_details['bsc_parameter_name']))
    
                        #remove all files currently in the directory
                        for zippath in glob.iglob(os.path.join(temp_dir, '*.nc')):
                            os.remove(zippath)
                        for zippath in glob.iglob(os.path.join(temp_dir, '*.tmp')):
                            os.remove(zippath)

def check_GHOST_stage_arguments_logs_submit(run_details):

    '''clean all existing argument/log/submit directories for defined GHOST stages to process.
       if directories do not already exist, create them.'''

    #iterate through defined GHOST stages to process
    for GHOST_stage in run_details['stages_to_process']:

        #define directory where submittal logs will be written
        log_dir = run_details['process_root']+'/run/stage_logs/{}_{}/{}'.format(run_details['network'],run_details['parameter'], GHOST_stage)

        #check if need to create any directories in path
        check_directory_existence(log_dir, directories_not_to_test=run_details['process_root'])
    
        #clean log directory
        for zippath in glob.iglob(os.path.join(log_dir, '*.out')):
            os.remove(zippath)
        for zippath in glob.iglob(os.path.join(log_dir, '*.err')):
            os.remove(zippath)

        #-----------------------------#

        #define directory where job arguments will be written
        arguments_dir = run_details['process_root']+'/run/stage_arguments/{}_{}'.format(run_details['network'],run_details['parameter'])

        #check if need to create any directories in path
        check_directory_existence(arguments_dir, directories_not_to_test='{}/run'.format(run_details['process_root']))

        #clean arguments directory 
        for zippath in glob.iglob(os.path.join(arguments_dir, '*.txt')):
            os.remove(zippath)
        for zippath in glob.iglob(os.path.join(arguments_dir, '*.grz')):
            os.remove(zippath)

        #-----------------------------#

        #define directory where jobs a from 
        submit_dir = run_details['process_root']+'/run/submit_stage/{}_{}'.format(run_details['network'],run_details['parameter'])

        #check if need to create any directories in path
        check_directory_existence(submit_dir, directories_not_to_test='{}/run'.format(run_details['process_root']))

        #clean submit directory 
        for zippath in glob.iglob(os.path.join(submit_dir, '*.sh')):
            os.remove(zippath)
        for zippath in glob.iglob(os.path.join(submit_dir, '*.rst')):
            os.remove(zippath)
        for zippath in glob.iglob(os.path.join(submit_dir, '*.log')):
            os.remove(zippath)


def write_station_specific_numpy(metadata,data,run_details,parameter_details,update_file=True):

    '''write out metadata/data to station specific numpy .npz file
       update_file variable is used to overwrite file when want to update it
    '''

    #get station reference
    station_reference = metadata['station_reference'][0]

    #get start and end of data ranges from file
    start_time = data['time'][0]
    end_time = data['time'][-1]+int(data['temporal_resolution'][-1])

    #deal with instances where temporary station file already exists and are not updating file - write dupN suffix variable for differentiation
    if not update_file:
        temporary_file_exists = os.path.isfile(run_details['data_root']+'{}/{}/native/{}/temporary_{}/{}_{}_{}.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],run_details['GHOST_stage'],station_reference,start_time,end_time))

        if temporary_file_exists:
            dup_n = 1
            while temporary_file_exists:
                if not os.path.isfile(run_details['data_root']+'{}/{}/native/{}/temporary_{}/{}_{}_{}:dup{}.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],run_details['GHOST_stage'],station_reference,start_time,end_time,dup_n)):
                    break
                dup_n +=1
            fname = run_details['data_root']+'{}/{}/native/{}/temporary_{}/{}_{}_{}:dup{}.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],run_details['GHOST_stage'],station_reference,start_time,end_time,dup_n)
            np.savez(fname,data=data,metadata=metadata)
            #give 770 permissions for file and make owner bsc32
            set_file_permissions_ownership(fname)
            return

    #write file for the first time or update it
    fname = run_details['data_root']+'{}/{}/native/{}/temporary_{}/{}_{}_{}.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],run_details['GHOST_stage'],station_reference,start_time,end_time)
    np.savez(fname,data=data,metadata=metadata)
    #give 770 permissions for file and make owner bsc32
    set_file_permissions_ownership(fname)
    return


def clean_native_directory(run_details, parameter_details, remove_lock=False):
        
    '''remove all existing station specific native resolution .npz files in native file directory for a specific network/parameter'''

    #set native directory for specific network/parameter
    native_dir = run_details['data_root'] + '{}/{}/native/{}/'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'])
    #iterate through all files in directory, removing each of them
    for zippath in glob.iglob(os.path.join(native_dir, '*.lock')):
        os.remove(zippath)
    if not remove_lock:
        for zippath in glob.iglob(os.path.join(native_dir, '*.npz')):
            os.remove(zippath)

def clean_output_resolution_directories(run_details, parameter_details):
    
    '''remove all existing final output resolution netCDF files for a specific network/parameter, for all defined output resolutions'''

    from GHOST_standards import standard_temporal_resolutions

    #iterate through all defined temporal resolutions to process
    if run_details['temporal_resolutions'] == 'default':
        temporal_resolutions = get_default_temporal_resolutions_per_matrix(parameter_details['matrix'])
    else:
        temporal_resolutions = run_details['temporal_resolutions']
    for temporal_resolution_code in temporal_resolutions:
    
        #get temporal resolution specific variables
        temporal_resolution_vars = standard_temporal_resolutions[temporal_resolution_code]

        #iterate through all regions to process:
        for region in run_details['regions']:

            #set network directory per region
            if region == 'GLOBAL':
                network_dir = run_details['network']
            else:
                network_dir = '{}_{}'.format(run_details['network'], region)

            #set output resolution directory for specific network/output resolution/parameter
            output_resolution_dir = run_details['data_root'] + '{}/{}/{}/{}/'.format(network_dir,run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],parameter_details['bsc_parameter_name'])
            #iterate through all files in directory, removing each of them
            for zippath in glob.iglob(os.path.join(output_resolution_dir, '*.nc')):
                os.remove(zippath)
            for zippath in glob.iglob(os.path.join(output_resolution_dir, '*.tmp')):
                os.remove(zippath)


def get_data_files(run_details, parameter_details, specific_station='', specific_year='', specific_month='', specific_day=''):
    '''gets list of all relevant data files for network/parameter (and an optional specific station/year/month/day) combination in set date range'''

    from dateutil.relativedelta import relativedelta

    # create datetime objects of defined start date/end date.
    start_date_datetime = datetime.datetime(int(run_details['start_year']), int(run_details['start_month']), 1)
    end_date_datetime = datetime.datetime(int(run_details['end_year']), int(run_details['end_month']), 1)

    # create empty list to store valid data files
    valid_data_files = []

    if 'AERONET' in run_details['network']:

        #check if are checking a getting files for a specific aeronet matrix, or all of them
        if 'current_aeronet_matrix' in list(parameter_details.keys()):
                data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/*.html'.format(run_details['network'],parameter_details['current_aeronet_matrix']))
        else:
            data_files = []
            for current_aeronet_matrix in parameter_details['aeronet_matrix']:
                data_files.extend(glob.glob(run_details['data_root']+'{}/original_files/{}/*.html'.format(run_details['network'],current_aeronet_matrix)))


        #create lists for storing all valid data file days/months/years
        valid_data_days = []
        valid_data_months = []
        valid_data_years = []
        
        # limit data files to just those with data inside defined data range
        for file_ii, data_file in enumerate(data_files):    
            file_data_day = data_file.split('/')[-1].replace('.html','')
            current_year = int(file_data_day[:4])
            current_month = int(file_data_day[4:6])
            current_day = int(file_data_day[6:8])
            start_date = datetime.datetime(current_year, current_month, current_day, 0, 0)
            end_date = datetime.datetime(current_year, current_month, current_day, 0, 0)  + datetime.timedelta(days=1)  
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])
                valid_data_days.append(file_data_day)
                valid_data_months.append(file_data_day[:6])
                valid_data_years.append(file_data_day[:4])


    elif run_details['network'] == 'BJMEMC':

        #get data files 
        if run_details['parameter'] in ['PM10','PM2.5']:
            data_files = sorted(glob.glob(run_details['data_root']+'{}/original_files/all/*.csv'.format(run_details['network'])))
        elif run_details['parameter'] in ['O3','NO2','SO2','CO']:
            data_files = sorted(glob.glob(run_details['data_root']+'{}/original_files/extra/*.csv'.format(run_details['network'])))

        #create lists for storing all valid data file days/months/years
        valid_data_days = []
        valid_data_months = []
        valid_data_years = []
        
        # limit data files to just those with data inside defined data range
        for file_ii, data_file in enumerate(data_files):    
            file_data_day = data_file.split('/')[-1].replace('.csv','')
            current_year = int(file_data_day[:4])
            current_month = int(file_data_day[4:6])
            current_day = int(file_data_day[6:8])
            start_date = datetime.datetime(current_year, current_month, current_day, 0, 0)
            end_date = datetime.datetime(current_year, current_month, current_day, 0, 0)  + datetime.timedelta(days=1)  
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])
                valid_data_days.append(file_data_day)
                valid_data_months.append(file_data_day[:6])
                valid_data_years.append(file_data_day[:4])


    elif run_details['network'] == 'CANADA_NAPS':

        #continuous parameters
        if (run_details['parameter'] == 'O3') or (run_details['parameter'] == 'NO') or (run_details['parameter'] == 'NO2') or (run_details['parameter'] == 'CO'):
            data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/*.csv'.format(run_details['network'],run_details['parameter']))
            stems = []
        #continuous and integrated parameters
        #SO2
        elif run_details['parameter'] in ['SO2']:
            data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/*.csv'.format(run_details['network'],run_details['parameter']))
            stem1 = run_details['data_root']+'{}/original_files/PMSpeciation/'.format(run_details['network'])
            stem2 = run_details['data_root']+'{}/original_files/IntegratedPM2.5/'.format(run_details['network']) 
            stems = [stem1,stem2]
        #PM10
        elif run_details['parameter'] in ['PM10']:
            data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/*.csv'.format(run_details['network'],run_details['parameter']))
            stem1 = run_details['data_root']+'{}/original_files/PMDICHOT/'.format(run_details['network'])
            stem2 = run_details['data_root']+'{}/original_files/IntegratedPM2.5-10/'.format(run_details['network'])
            stems = [stem1,stem2]            
        #PM2.5
        elif run_details['parameter'] in ['PM2.5']:
            data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/*.csv'.format(run_details['network'],run_details['parameter']))
            stem1 = run_details['data_root']+'{}/original_files/PMDICHOT/'.format(run_details['network'])
            stem2 = run_details['data_root']+'{}/original_files/PartisolPM2.5/'.format(run_details['network'])
            stem3 = run_details['data_root']+'{}/original_files/IntegratedPM2.5/'.format(run_details['network'])
            stem4 = run_details['data_root']+'{}/original_files/NAPSReferenceMethodPM2.5/'.format(run_details['network'])
            stems = [stem1,stem2,stem3,stem4]
        #integrated parameters
        #NH3, HNO3
        elif run_details['parameter'] in ['NH3','HNO3']:
            data_files = []
            stem1 = run_details['data_root']+'{}/original_files/PMSpeciation/'.format(run_details['network'])
            stem2 = run_details['data_root']+'{}/original_files/IntegratedPM2.5/'.format(run_details['network']) 
            stems = [stem1,stem2]
        #HCHO
        elif run_details['parameter'] in ['HCHO','ACETALDEHYDE']:
            data_files = []
            stems = [run_details['data_root']+'{}/original_files/Carbonyls/'.format(run_details['network'])]
        #VOC
        elif run_details['parameter'] in ['CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE']:
            data_files = []
            stems = [run_details['data_root']+'{}/original_files/VOC/'.format(run_details['network'])]
        #speciated pm 
        elif parameter_details['matrix'] == 'pm':
            data_files = []
            stems = [run_details['data_root']+'{}/original_files/PMSpeciation/'.format(run_details['network'])]
        #speciated pm10
        elif parameter_details['matrix'] == 'pm10':
            data_files = []
            stem1 = run_details['data_root']+'{}/original_files/PMDICHOT/'.format(run_details['network'])
            stem2 = run_details['data_root']+'{}/original_files/IntegratedPM2.5-10/'.format(run_details['network'])
            stem3 = run_details['data_root']+'{}/original_files/PMSpeciation/'.format(run_details['network'])
            stems = [stem1,stem2,stem3]
        #speciated pm2.5
        elif parameter_details['matrix'] == 'pm2.5':
            data_files = []
            stem1 = run_details['data_root']+'{}/original_files/PMDICHOT/'.format(run_details['network'])
            stem2 = run_details['data_root']+'{}/original_files/PartisolPM2.5/'.format(run_details['network'])
            stem3 = run_details['data_root']+'{}/original_files/IntegratedPM2.5/'.format(run_details['network'])
            stem4 = run_details['data_root']+'{}/original_files/NAPSReferenceMethodPM2.5/'.format(run_details['network'])
            stem5 = run_details['data_root']+'{}/original_files/PMSpeciation/'.format(run_details['network'])
            stems = [stem1,stem2,stem3,stem4,stem5]

        #iterate through stems and get integrated files
        for stem in stems:
            for root, dirs, files in os.walk(stem):
                for f in files:
                    if (f.endswith("_EN.xls")) or (f.endswith("_EN.xlsx")):     
                        data_files.append(os.path.join(root, f))
                    elif (f.endswith("_EN.XLS")) or (f.endswith("_EN.XLSX")):       
                        data_files.append(os.path.join(root, f))
                    elif (f.endswith(".xls")) or (f.endswith(".xlsx")):
                        if (not f.endswith("FR.xls")) & (not f.endswith("FR.xlsx")):
                            if f.endswith("_CARB.xls"):
                                if run_details['parameter'] in ['C','EC','OC']:                             
                                    data_files.append(os.path.join(root, f))
                            elif f.endswith("_IC.xls"):
                                if run_details['parameter'] in ['Ca++','Cl-','NH4+','K+','Mg++','Mn','MSA','Na+','NO3-','SO4--']:
                                    data_files.append(os.path.join(root, f)) 
                            elif f.endswith("_NA.xls"):
                                if run_details['parameter'] in ['NO3-','SO4--']:
                                    data_files.append(os.path.join(root, f)) 
                            elif f.endswith("_NH4.xls"):
                                if run_details['parameter'] in ['NH4+']:
                                    data_files.append(os.path.join(root, f)) 
                            elif f.endswith("_ICPMS.xls"):
                                if run_details['parameter'] in ['PM10_As','PM10_Cd','PM10_Co','PM10_Cr','PM10_Cu','PM10_Fe','PM10_Mn','PM10_Ni','PM10_Pb','PM10_Se','PM10_V','PM10_Zn','PM2.5_As','PM2.5_Cd','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_Fe','PM2.5_Mn','PM2.5_Ni','PM2.5_Pb','PM2.5_Se','PM2.5_V','PM2.5_Zn']:
                                    data_files.append(os.path.join(root, f))
                            elif f.endswith("_SPEC.xls"):
                                if run_details['parameter'] in ['SO2','NH3','HNO3','EC','NH4+','NO3-','OC','SO4--']:
                                    data_files.append(os.path.join(root, f))
                            elif (f.endswith("_WICPMS.xls")) or (f.endswith("_LEV.xls")) or (f.endswith("_VOCS.xls")):
                                continue
                            else:
                                data_files.append(os.path.join(root, f))
                    elif (f.endswith(".XLS")) or (f.endswith(".XLSX")):
                        if (not f.endswith("FR.XLS")) & (not f.endswith("FR.XLSX")):
                            if f.endswith("_CARB.XLS"):
                                if run_details['parameter'] in ['C','EC','OC']:                             
                                    data_files.append(os.path.join(root, f))
                            elif f.endswith("_IC.XLS"):
                                if run_details['parameter'] in ['Ca++','Cl-','NH4+','K+','Mg++','Mn','MSA','Na+','NO3-','SO4--']:
                                    data_files.append(os.path.join(root, f)) 
                            elif f.endswith("_NA.XLS"):
                                if run_details['parameter'] in ['NO3-','SO4--']:
                                    data_files.append(os.path.join(root, f)) 
                            elif f.endswith("_NH4.XLS"):
                                if run_details['parameter'] in ['NH4+']:
                                    data_files.append(os.path.join(root, f)) 
                            elif f.endswith("_ICPMS.XLS"):
                                if run_details['parameter'] in ['PM10_As','PM10_Cd','PM10_Co','PM10_Cr','PM10_Cu','PM10_Fe','PM10_Mn','PM10_Ni','PM10_Pb','PM10_Se','PM10_V','PM10_Zn','PM2.5_As','PM2.5_Cd','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_Fe','PM2.5_Mn','PM2.5_Ni','PM2.5_Pb','PM2.5_Se','PM2.5_V','PM2.5_Zn']:
                                    data_files.append(os.path.join(root, f))
                            elif f.endswith("_SPEC.XLS"):
                                if run_details['parameter'] in ['SO2','NH3','HNO3','EC','NH4+','NO3-','OC','SO4--']:
                                    data_files.append(os.path.join(root, f))
                            elif (f.endswith("_WICPMS.XLS")) or (f.endswith("_LEV.XLS")) or (f.endswith("_VOCS.XLS")):
                                continue
                            else:
                                data_files.append(os.path.join(root, f))
                
        #list to append valid data file years
        valid_data_years = []

        #limit data files to just those with data inside defined data range
        for data_file in data_files:
            if data_file.endswith(".csv"):
                data_year = int(data_file.split('_')[-1][:4])
            else:
                data_year = int(data_file.split('/')[-2])
            start_date = datetime.datetime(data_year, 1, 1, 0, 0)
            end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_file)
                valid_data_years.append(str(data_year))


    elif run_details['network'] == 'CAPMoN':
        #O3
        if run_details['parameter'] == 'O3':        
            data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/*.csv'.format(run_details['network'],run_details['parameter']))

            #list to append valid data file years
            valid_data_years = []
        
            # limit data files to just those with data inside defined data range
            for data_file in data_files:
                data_year = int(data_file.split('/')[-1].split('-')[-1][:4])
                start_date = datetime.datetime(data_year, 1, 1, 0, 0)
                end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
                if (start_date_datetime <= start_date < end_date_datetime) or (
                        start_date_datetime < end_date <= end_date_datetime) or (
                        start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                    valid_data_files.append(data_file)
                    valid_data_years.append(str(data_year))

        #particulate metals
        else:
            data_files = glob.glob(run_details['data_root']+'{}/original_files/Particulate_Metals/*.csv'.format(run_details['network']))
            valid_data_files = [data_files[0]]
            valid_data_years = ['all']
                    

    elif run_details['network'] == 'CHILE_SINCA':
        stem = run_details['data_root'] + '{}/original_files/'.format(run_details['network'])
        valid_data_files = []

        if run_details['parameter'] in ['PM10_As','PM10_Cu','PM10_Pb']:
            chile_param = run_details['parameter'].split('_')[-1]
        else:
            chile_param = copy.deepcopy(run_details['parameter'])

        for root, dirs, files in os.walk(stem):
            for f in files:
                if (f.endswith("{}_hourly.csv".format(chile_param))) or (f.endswith("{}_daily.csv".format(chile_param))) or (f.endswith("{}_monthly.csv".format(chile_param))):
                    valid_data_files.append(os.path.join(root, f))

        #get station references from all valid data files
        file_station_references = [f.split('/')[-2] for f in valid_data_files]


    elif run_details['network'] == 'CNEMC':

        #get data files 
        data_files = sorted(glob.glob(run_details['data_root']+'{}/original_files/sites/*.csv'.format(run_details['network'])))

        #create lists for storing all valid data file days/months/years
        valid_data_days = []
        valid_data_months = []
        valid_data_years = []
        
        # limit data files to just those with data inside defined data range
        for file_ii, data_file in enumerate(data_files):    
            file_data_day = data_file.split('/')[-1].replace('.csv','')
            current_year = int(file_data_day[:4])
            current_month = int(file_data_day[4:6])
            current_day = int(file_data_day[6:8])
            start_date = datetime.datetime(current_year, current_month, current_day, 0, 0)
            end_date = datetime.datetime(current_year, current_month, current_day, 0, 0)  + datetime.timedelta(days=1)  
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])
                valid_data_days.append(file_data_day)
                valid_data_months.append(file_data_day[:6])
                valid_data_years.append(file_data_day[:4])


    elif run_details['network'] == 'EANET':

        #process all files except WET DEP 
        stem = run_details['data_root'] + '{}/original_files/'.format(run_details['network'])
        data_files = []
        for root, dirs, files in os.walk(stem):
            for f in files:
                if (f.endswith("AT.csv")) or (f.endswith("FP.csv")) or (f.endswith("PS.csv")):
                    data_files.append(os.path.join(root, f))

        # get start and end timepoints of data files
        data_files_start_dates = [datetime.datetime(int(data_file.split('/')[-1].split('_')[-1][:4]), 1, 1) for
                                  data_file in data_files]
        for file_ii, start_date in enumerate(data_files_start_dates):
            end_date = start_date + relativedelta(years=1)

            # limit data files to just those with data inside defined data range
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (
                    start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])

        #get station references from all valid data files
        file_station_references = [f.split('/')[-1].split('_')[0] for f in valid_data_files]


    elif run_details['network'] == 'EBAS':

        # PM10, PM2.5, PM1
        if run_details['parameter'] in ['PM10','PM2.5','PM1']:
            data_files = glob.glob(
                run_details['data_root'] + '{}/original_files/aerosol/{}/mass/*.nas'.format(run_details['network'],
                                                                                            run_details['parameter']))           
        # bulk optical/speciated
        elif parameter_details['matrix'] == 'pm':
            
            #optical coefficients
            if '_COEFF_' in run_details['parameter']:
                param = '_'.join(run_details['parameter'].split('_')[:-1])
                data_files = glob.glob(
                    run_details['data_root'] + '{}/original_files/aerosol/bulk/optical/{}/*.nas'.format(
                        run_details['network'], param))
            #speciated
            else:
                data_files = glob.glob(
                    run_details['data_root'] + '{}/original_files/aerosol/bulk/speciated/{}/*.nas'.format(
                        run_details['network'], run_details['parameter']))
                #also extend data files list with corrected component files (C, OC)
                if run_details['parameter'] in ['C','OC']:
                    data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/aerosol/bulk/speciated/{}_corrected/*.nas'.format(
                        run_details['network'], run_details['parameter'])))

        # PM10/PM2.5/PM1 optical/speciated
        elif parameter_details['matrix'] in ['pm10','pm2.5','pm1']:
            param_split = run_details['parameter'].split('_')
            aerosol_type = param_split[0]
            param = '_'.join(param_split[1:])

            #optical coefficients
            if '_COEFF_' in run_details['parameter']:
                param_coeff = '_'.join(param.split('_')[:-1])
                data_files = glob.glob(run_details['data_root'] + '{}/original_files/aerosol/{}/optical/{}/*.nas'.format(
                    run_details['network'], aerosol_type, param_coeff)) 
            #speciated
            else:
                data_files = glob.glob(run_details['data_root'] + '{}/original_files/aerosol/{}/speciated/{}/*.nas'.format(
                    run_details['network'], aerosol_type, param)) 
                #also extend data files list with corrected component files (C, OC)
                if run_details['parameter'] in ['PM10_C','PM10_OC','PM2.5_C','PM2.5_OC','PM1_C','PM1_OC']:
                    data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/aerosol/{}/speciated/{}_corrected/*.nas'.format(
                    run_details['network'], aerosol_type, param))) 

        #aerosol optical depth
        elif parameter_details['matrix'] == 'aod':
            data_files = glob.glob(
                run_details['data_root'] + '{}/original_files/aerosol/bulk/optical/AOD/*.nas'.format(
                    run_details['network']))

        # all other parameters
        else:
            data_files = glob.glob(
                run_details['data_root'] + '{}/original_files/{}/{}/*.nas'.format(run_details['network'],
                                                                                parameter_details['matrix'],
                                                                                run_details['parameter']))

        # get start and end timepoints of data files
        data_files_without_root = [data_file.split('/')[-1] for data_file in data_files]
        data_files_start_dates = [data_file.split('.')[1] for data_file in data_files_without_root]
        data_files_start_dates = [
            datetime.datetime(int(start_date[:4]), int(start_date[4:6]), int(start_date[6:8]), int(start_date[8:10]),
                              int(start_date[10:12])) for start_date in data_files_start_dates]
        data_files_file_lengths = [data_file.split('.')[6] for data_file in data_files_without_root]
        for file_ii, start_date in enumerate(data_files_start_dates):
            file_length_int = int(re.findall(r'\d+', data_files_file_lengths[file_ii])[0])
            file_length_str = re.findall(r'\D+', data_files_file_lengths[file_ii])[0]
            if file_length_str == 'h':
                end_date = start_date + relativedelta(hours=file_length_int)
            elif file_length_str == 'd':
                end_date = start_date + relativedelta(days=file_length_int)
            elif file_length_str == 'w':
                end_date = start_date + relativedelta(weeks=file_length_int)
            elif file_length_str == 'mo':
                end_date = start_date + relativedelta(months=file_length_int)
            elif file_length_str == 'y':
                end_date = start_date + relativedelta(years=file_length_int)

            # limit data files to just those with data inside defined data range
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])

        #get station references from all valid data files
        file_station_references = [f.split('/')[-1].split('.')[0] for f in valid_data_files] 


    elif 'EEA_AIRBASE' in run_details['network']:
        stem = run_details['data_root'] + '{}/original_files'.format(run_details['network'])

        # gather all data files
        data_files = []
        for root, dirs, files in os.walk(stem):
            for f in files:
                if (not f.endswith('.csv')) & (not f.endswith('.xml')) & (not f.endswith('.kmz')):
                    if ('hour8' not in f) & ('dymax' not in f):
                        data_files.append(os.path.join(root, f))

        #limit files to those for current component
        data_files = [data_file for data_file in data_files if data_file.split('/')[-1][7:12] == parameter_details['airbase_code']]

        # get start and end timepoints of data files
        data_files_start_dates = [datetime.datetime(int(data_file.split('.')[-2].split('-')[2]), int(data_file.split('.')[-2].split('-')[1]), int(data_file.split('.')[-2].split('-')[0])) for data_file in data_files]
        data_files_end_dates = [datetime.datetime(int(data_file.split('.')[-1].split('-')[2]), int(data_file.split('.')[-1].split('-')[1]), int(data_file.split('.')[-1].split('-')[0])) for data_file in data_files]
        for file_ii, (start_date, end_date) in enumerate(zip(data_files_start_dates, data_files_end_dates)):
            # limit data files to just those with data inside defined data range
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (
                    start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])

        #get station references from all valid data files
        f_split = [f.split('/')[-1] for f in valid_data_files]
        file_station_references = [spl[:7] for spl in f_split]


    elif 'EEA_AQ_eReporting' in run_details['network']:
        # gather data files from both E1a/E2a streams

        # PM10, PM2.5, PM1
        if run_details['parameter'] in ['PM10','PM2.5','PM1']:
            stem_E1a = run_details['data_root'] + '{}/original_files/E1a/aerosol/{}/mass'.format(run_details['network'],run_details['parameter'])
            stem_E2a = run_details['data_root'] + '{}/original_files/E2a/aerosol/{}/mass'.format(run_details['network'],run_details['parameter'])
        # bulk speciated
        elif parameter_details['matrix'] == 'pm':
            stem_E1a = run_details['data_root'] + '{}/original_files/E1a/aerosol/bulk/speciated/{}'.format(run_details['network'], run_details['parameter'])
            stem_E2a = run_details['data_root'] + '{}/original_files/E2a/aerosol/bulk/speciated/{}'.format(run_details['network'], run_details['parameter'])        
        # PM10/PM2.5/PM1 speciated
        elif parameter_details['matrix'] in ['pm10','pm2.5','pm1']:
            param_split = run_details['parameter'].split('_')
            aerosol_type = param_split[0]
            param = '_'.join(param_split[1:])
            stem_E1a = run_details['data_root'] + '{}/original_files/E1a/aerosol/{}/speciated/{}'.format(run_details['network'], aerosol_type, param)
            stem_E2a = run_details['data_root'] + '{}/original_files/E2a/aerosol/{}/speciated/{}'.format(run_details['network'], aerosol_type, param)
        # all other parameters
        else:
            stem_E1a = run_details['data_root'] + '{}/original_files/E1a/{}/{}'.format(run_details['network'], parameter_details['matrix'], run_details['parameter'])
            stem_E2a = run_details['data_root'] + '{}/original_files/E2a/{}/{}'.format(run_details['network'], parameter_details['matrix'], run_details['parameter'])
                
        # gather all data files
        data_files = []
        for stem in [stem_E1a, stem_E2a]:
            for root, dirs, files in os.walk(stem):
                for f in files:
                    if f.endswith("timeseries.csv"):
                        data_files.append(os.path.join(root, f))
                
        # get start and end timepoints of data files
        data_files_start_dates = [datetime.datetime(int(data_file.split('/')[-1].split('_')[3]), 1, 1) for
                                  data_file in data_files]
        for file_ii, start_date in enumerate(data_files_start_dates):
            end_date = start_date + relativedelta(years=1)

            # limit data files to just those with data inside defined data range
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (
                    start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])

        #get station references from all valid data files
        f_split = [f.split('/')[-1].split('_') for f in valid_data_files]
        file_station_references = ['{}_{}_{}'.format(spl[0],spl[1],spl[2]) for spl in f_split]

    elif 'JAPAN_NIES' in run_details['network']:  

        #list to append valid data file years
        valid_data_years = []
        
        #set file stems
        hourly_stem =  run_details['data_root'] + '{}/original_files/hourly'.format(run_details['network'])
        monthly_stem =  run_details['data_root'] + '{}/original_files/monthly'.format(run_details['network'])
        bureau_stem =  run_details['data_root'] + '{}/original_files/bureau'.format(run_details['network'])

        #iterate through parameter codes to get hourly/monthly files
        for parameter_code in parameter_details['japan_nies_code']:

            #set file determinant per stem
            hourly_det = '_{}.txt'.format(parameter_code)
            monthly_det = '{}00.txt'.format(parameter_code)
            
            #gather files
            for stem, file_det in zip([hourly_stem,monthly_stem], [hourly_det,monthly_det]):
                for root, dirs, files in os.walk(stem):
                    for f in files:
                        if f.endswith(file_det):      
                            data_file = os.path.join(root, f)
                            data_year = int(data_file.split('/')[-2])
                            start_date = datetime.datetime(data_year, 1, 1, 0, 0)
                            end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
                            if (start_date_datetime <= start_date < end_date_datetime) or (
                                    start_date_datetime < end_date <= end_date_datetime) or (
                                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                                valid_data_files.append(data_file)
                                valid_data_years.append(str(data_year))

        #gather bureau files
        bureau_det = '.csv'
        for root, dirs, files in os.walk(bureau_stem):
            for f in files:
                if f.endswith(bureau_det):      
                    data_file = os.path.join(root, f)
                    data_year = int(data_file.split('/')[-2])
                    start_date = datetime.datetime(data_year, 1, 1, 0, 0)
                    end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
                    if (start_date_datetime <= start_date < end_date_datetime) or (
                            start_date_datetime < end_date <= end_date_datetime) or (
                            start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                        valid_data_files.append(data_file)
                        valid_data_years.append(str(data_year))


    elif 'MEXICO_CDMX' in run_details['network']:    
        data_files = glob.glob(run_details['data_root'] + '{}/original_files/*.CSV'.format(run_details['network']))

        #list to append valid data file years
        valid_data_years = []

        #limit data files to just those with data inside defined data range
        for data_file in data_files:
            data_year = int(data_file.split('_')[-1][:4])
            start_date = datetime.datetime(data_year, 1, 1, 0, 0)
            end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_file)
                valid_data_years.append(str(data_year))


    elif 'MITECO' in run_details['network']:
        if run_details['parameter'] in ['PM10_As','PM10_BaP','PM10_Cd','PM10_Ni','PM10_Pb']:
            if run_details['parameter'] == 'PM10_BaP':
                param = 'B(a)P'
            else:
                param = run_details['parameter'].split('_')[-1]
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/{}_*.csv'.format(run_details['network'],param))
        elif run_details['parameter'] in ['BENZENE','TOLUENE']:
            param = copy.deepcopy(parameter_details['chemical_formula'])
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/{}_*.csv'.format(run_details['network'],param))
        elif run_details['parameter'] == 'XYLENE':
            param = 'XYL'
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/{}_*.csv'.format(run_details['network'],param))
        else:    
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/{}_*.csv'.format(run_details['network'],run_details['parameter']))

        #list to append valid data file years
        valid_data_years = []

        #limit data files to just those with data inside defined data range
        for data_file in data_files:
            data_year = int(data_file.split('_')[-1][:4])
            start_date = datetime.datetime(data_year, 1, 1, 0, 0)
            end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_file)
                valid_data_years.append(str(data_year))
        

    elif 'NOAA_ISD' in run_details['network']:
        data_files = []
        stem = run_details['data_root'] + '{}/original_files/meteo'.format(run_details['network'])
        for root, dirs, files in os.walk(stem):
            for f in files:
                data_files.append(os.path.join(root, f))

        file_station_references = []
        for data_file in data_files:
            f_split = data_file.split('/')[-1].split('-')
            stationref = f_split[0] + f_split[1]
            stationyear = f_split[2]
            start_date = datetime.datetime(int(stationyear),1,1,0,0)
            end_date = datetime.datetime(int(stationyear)+1,1,1,0,0)

            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (
                    start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_file)
                file_station_references.append(stationref)

    elif run_details['network'] == 'UK_AIR':

        # get all data files
        data_files = []
        data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/AURN/*.csv'.format(run_details['network'])))

        if parameter_details['matrix'] in ['pm']:
            if run_details['parameter'] == 'Al':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Al.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'As':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/As.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Cd':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Cd.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Co':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Co.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Cr':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Cr.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Cu':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Cu.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Fe':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Fe.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Mn':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Mn.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Ni':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Ni.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Pb':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Pb.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Se':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Se.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'V':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/V.csv'.format(run_details['network']))
            elif run_details['parameter'] == 'Zn':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/Zn.csv'.format(run_details['network']))

        elif parameter_details['matrix'] == 'pm10':
            if run_details['parameter'] == 'PM10':
                data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_PM/*.csv'.format(run_details['network'])))
            else:
                data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_PM10_Speciated/*.csv'.format(run_details['network'])))

        elif parameter_details['matrix'] == 'pm2.5':
            if run_details['parameter'] == 'PM2.5':
                data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_PM/*.csv'.format(run_details['network'])))
            else:
                data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_PM2.5_Speciated/*.csv'.format(run_details['network'])))
                if run_details['parameter'] == 'PM2.5_C':
                    data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/PM2.5_C.csv'.format(run_details['network']))
                elif run_details['parameter'] == 'PM2.5_EC':
                    data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/PM2.5_EC_reflectance.csv'.format(run_details['network']))
                    data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/PM2.5_EC_transmittance.csv'.format(run_details['network']))
                elif run_details['parameter'] == 'PM2.5_OC':
                    data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/PM2.5_OC_reflectance.csv'.format(run_details['network']))
                    data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/PM2.5_OC_transmittance.csv'.format(run_details['network']))

        elif parameter_details['matrix'] == 'pm1':
            if run_details['parameter'] == 'PM1':
                data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_PM/*.csv'.format(run_details['network'])))
            else:
                data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_PM1_Speciated/*.csv'.format(run_details['network'])))

        elif parameter_details['matrix'] == 'gas':
            data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_Other_Gases/*.csv'.format(run_details['network'])))            

            if run_details['parameter'] in ['ACETYLENE','ETHENE','ETHANE','PROPENE','PROPANE','1,3-BUTADIENE','1-BUTENE','ISOBUTANE','ISOPRENE','N-PENTANE','BENZENE','N-HEXANE','TOLUENE','XYLENE','1,2,4-TRIMETHYLBENZENE']:
                data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/Auto_VOC/*.csv'.format(run_details['network'])))
                if run_details['parameter'] == 'BENZENE':
                    data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/C6H6.csv'.format(run_details['network']))
            
            elif run_details['parameter'] == 'NH3':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/NH3_active.csv'.format(run_details['network']))
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/NH3_diffusiontube.csv'.format(run_details['network']))
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/NH3_passive.csv'.format(run_details['network']))

            elif run_details['parameter'] == 'SO2':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/SO2.csv'.format(run_details['network']))

            elif run_details['parameter'] == 'HCl':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/HCl.csv'.format(run_details['network']))

            elif run_details['parameter'] == 'HNO3':
                data_files.append(run_details['data_root'] + '{}/original_files/Non-Auto/HNO3.csv'.format(run_details['network']))

        #list to append valid data file years
        valid_data_years = []

        #limit data files to just those with data inside defined data range
        for data_file in data_files:
            if ('/AURN/' in data_file) or ('/Auto_Other_Gases/' in data_file):
                data_year = int(data_file.split('.csv')[-2][-4:])
                start_date = datetime.datetime(data_year, 1, 1, 0, 0)
                end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
            elif ('/Auto_PM/' in data_file) or ('/Auto_PM10_Speciated/' in data_file) or ('/Auto_PM2.5_Speciated/' in data_file) or ('/Auto_PM1_Speciated/' in data_file) or ('/Auto_VOC/' in data_file):
                start_data_year = int(data_file.split('/')[-1][:4])
                end_data_year = int(data_file.split('/')[-1][5:9])
                start_date = datetime.datetime(start_data_year, 1, 1, 0, 0)
                end_date = datetime.datetime(end_data_year, 1, 1, 0, 0)
            elif '/Non-Auto/' in data_file:
                valid_data_files.append(data_file)
                valid_data_years.append('all')
                continue
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_file)
                valid_data_years.append(str(data_year))

    elif run_details['network'] == 'US_EPA_AirNow_DOS':

        if run_details['parameter'] == 'O3':
            param = 'OZONE'
        else:
            param = copy.deepcopy(run_details['parameter'])

        #get data files
        data_files = sorted(glob.glob(run_details['data_root'] + '{}/original_files/*_{}_*.csv'.format(run_details['network'], param)))

        # get start and end timepoints of data files
        data_files_start_dates = [datetime.datetime(int(data_file.split('/')[-1].split('_')[2][:4]), 1, 1) for
                                  data_file in data_files]
        for file_ii, start_date in enumerate(data_files_start_dates):
            end_date = start_date + relativedelta(years=1)

            # limit data files to just those with data inside defined data range
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (
                    start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_files[file_ii])

        #get station references from all valid data files
        file_station_references = [f.split('/')[-1].split('_')[0] for f in valid_data_files]


    elif run_details['network'] == 'US_EPA_AQS':
        #O3/ NO / NO2 / CO / SO2 - only take hourly files as daily files have no methods (but same number of stations between resolutions)
        if run_details['parameter'] in ['O3','NO','NO2','CO','SO2']:    
            data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/hourly*.csv'.format(run_details['network'],run_details['parameter']))
        #PM10 / PM2.5 
        elif run_details['parameter'] in ['PM10','PM2.5']:
            data_files = glob.glob(run_details['data_root']+'{}/original_files/{}/*.csv'.format(run_details['network'],run_details['parameter']))   
        #VOCs 
        elif run_details['parameter'] in ['CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE','NMVOC','VOC','HC','HCHO']: 
            data_files = glob.glob(run_details['data_root']+'{}/original_files/VOCs/*.csv'.format(run_details['network']))
        #Pb
        elif run_details['parameter'] in ['Pb','PM10_Pb','PM2.5_Pb']:        
            data_files = glob.glob(run_details['data_root']+'{}/original_files/Pb/*.csv'.format(run_details['network']))
            data_files.extend(glob.glob(run_details['data_root']+'{}/original_files/PM10_speciated/*.csv'.format(run_details['network'])))
            data_files.extend(glob.glob(run_details['data_root']+'{}/original_files/PM2.5_speciated/*.csv'.format(run_details['network'])))
        #speciated PM10
        elif parameter_details['matrix'] == 'pm10':
            data_files = glob.glob(run_details['data_root']+'{}/original_files/PM10_speciated/*.csv'.format(run_details['network']))
        #speciated PM2.5
        elif parameter_details['matrix'] == 'pm2.5':
            data_files = glob.glob(run_details['data_root']+'{}/original_files/PM2.5_speciated/*.csv'.format(run_details['network']))

        #list to append valid data file years
        valid_data_years = []
    
        # limit data files to just those with data inside defined data range
        for data_file in data_files:
            data_year = int(data_file.split('/')[-1].split('_')[-1][:4])
            start_date = datetime.datetime(data_year, 1, 1, 0, 0)
            end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
            if (start_date_datetime <= start_date < end_date_datetime) or (
                    start_date_datetime < end_date <= end_date_datetime) or (
                    start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                valid_data_files.append(data_file)
                valid_data_years.append(str(data_year))


    elif run_details['network'] == 'US_EPA_CASTNET':
        if run_details['parameter'] == 'O3':
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/O3/ozone_*.csv'.format(run_details['network']))
        elif run_details['parameter'] in ['NO','NO2','CO','NH3','TEMP_2M']:    
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/hourly_gas/hourly_gas_*.csv'.format(run_details['network']))
        elif run_details['parameter'] in ['Ca++','Cl-','K+','Mg++','Na+','NH4+','NO3-','SO4--']:
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/drychem/drychem.csv'.format(run_details['network']))
        elif run_details['parameter'] in ['PM2.5','PM2.5_Al','PM2.5_As','PM2.5_C','PM2.5_Ca++','PM2.5_Cd','PM2.5_Cl-','PM2.5_Co','PM2.5_Cr','PM2.5_Cu','PM2.5_EC','PM2.5_Fe','PM2.5_Hg','PM2.5_K+','PM2.5_Mg++','PM2.5_Mn','PM2.5_Na+','PM2.5_Ni','PM2.5_NO3-','PM2.5_OC','PM2.5_Pb','PM2.5_Se','PM2.5_SO4--','PM2.5_V','PM2.5_Zn']:
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/vischem/vischem.csv'.format(run_details['network']))
        elif run_details['parameter'] in ['SO2','HNO3']:
            data_files = glob.glob(run_details['data_root'] + '{}/original_files/hourly_gas/hourly_gas_*.csv'.format(run_details['network']))
            data_files.extend(glob.glob(run_details['data_root'] + '{}/original_files/drychem/drychem.csv'.format(run_details['network'])))
                
        #remove tableinfo and columninfo files
        data_files = [f for f in data_files if ('tableinfo' not in f) & ('columninfo' not in f)]
        
        #list to append valid data file years
        valid_data_years = []

        #list to append non-year specific drychem and vischem files
        non_year_files = []        

        #limit data files to just those with data inside defined data range
        #only check for O3 and hourly gas file types, return both drychem and vischem files if specific year is earliest valid year
        for data_file in data_files:
            if ('original_files/O3' in data_file) or ('original_files/hourly_gas' in data_file):
                data_year = int(data_file.split('_')[-1][:4])
                start_date = datetime.datetime(data_year, 1, 1, 0, 0)
                end_date = datetime.datetime(data_year+1, 1, 1, 0, 0)
                if (start_date_datetime <= start_date < end_date_datetime) or (
                        start_date_datetime < end_date <= end_date_datetime) or (
                        start_date <= start_date_datetime < end_date) or (start_date < end_date_datetime <= end_date):
                    valid_data_files.append(data_file)
                    valid_data_years.append(str(data_year))
            else:
                non_year_files.append(data_file)

        #if not getting files for specific year, add non year specific files
        if specific_year == '':
            valid_data_files.extend(non_year_files)
        #otherwise, if specific year is 'all' add non year specific files
        elif specific_year == 'all':
            valid_data_files.extend(non_year_files)
            valid_data_years.extend(['all']*len(non_year_files))

    elif run_details['network'] == 'US_NADP_AMNet':
        valid_data_files.append(run_details['data_root'] + '{}/original_files/AMNET-ALL-h.csv'.format(run_details['network']))


    elif run_details['network'] == 'US_NADP_AMoN':
        valid_data_files.append(run_details['data_root'] + '{}/original_files/AMoN-ALL-W-i.csv'.format(run_details['network']))


    elif run_details['network'] == 'WMO_WDCGG':
        data_files = []
        stem = run_details['data_root'] + '{}/original_files/{}'.format(run_details['network'], run_details['parameter'])
        for root, dirs, files in os.walk(stem):
            for f in files:
                if f.endswith(".txt"):
                    data_files.append(os.path.join(root, f))
 
        #remove met files/aircraft/ship files
        valid_data_files = [f for f in data_files if ('_met.txt' not in f) & ('_aircraft-' not in f) & ('_ship-' not in f)]

        #get station references from all valid data files
        file_station_references = [f.split('/')[-1].split('_')[1] for f in valid_data_files]



    # if a specific station variable has been defined, limit data files to be just those associated with the defined station reference
    if specific_station != '':
        valid_data_files = [data_file for data_file, file_station_reference in
                            zip(valid_data_files, file_station_references) if
                            str(specific_station) == str(file_station_reference)]
    # if a specific data year variable has been defined, limit data files to be just those associated with the defined data year
    if specific_year != '':
        valid_data_files = [data_file for data_file, file_data_year in zip(valid_data_files, valid_data_years) if
                            str(specific_year) == str(file_data_year)]

    # if a specific data month variable has been defined, limit data files to be just those associated with the defined data month
    if specific_month != '':
        valid_data_files = [data_file for data_file, file_data_month in zip(valid_data_files, valid_data_months) if
                            str(specific_month) == str(file_data_month)]

    # if a specific data day variable has been defined, limit data files to be just those associated with the defined data day  
    if specific_day != '':
        valid_data_files = [data_file for data_file, file_data_day in zip(valid_data_files, valid_data_days) if
                            str(specific_day) == str(file_data_day)]


    return sorted(valid_data_files)


def get_YYYY_from_filenames(data_files, run_details):

    '''get unique list of data years as YYYY strings to process from data filenames.
       process for this differs by network.
    ''' 

    data_years = []

    if 'AERONET' in run_details['network']:
        for i, f in enumerate(data_files):
            data_years.append(f.split('/')[-1][:4])
        data_years = np.unique(data_years)

    elif 'CANADA_NAPS' in run_details['network']:
        for i, f in enumerate(data_files):
            if f.endswith(".csv"):
                data_years.append(f.split('_')[-1][:4])
            else:
                data_years.append(f.split('/')[-2])
        data_years = np.unique(data_years)

    elif 'CAPMoN' in run_details['network']:
        for i, f in enumerate(data_files):
            if ('original_files/O3' in f):
                data_years.append(f.split('-')[-1][:4])
            else:
                data_years.append('all')
        data_years = np.unique(data_years)

    elif 'JAPAN_NIES' in run_details['network']:
        for i, f in enumerate(data_files):
            data_years.append(f.split('/')[-2])
        data_years = np.unique(data_years)
    
    elif 'MEXICO_CDMX' in run_details['network']:
        for i, f in enumerate(data_files):
            data_years.append(f.split('_')[-1][:4])
        data_years = np.unique(data_years)

    elif 'MITECO' in run_details['network']:
        for i, f in enumerate(data_files):
            data_years.append(f.split('_')[-1][:4])
        data_years = np.unique(data_years)

    elif 'UK_AIR' in run_details['network']:
        for i, f in enumerate(data_files):
            if ('/AURN/' in f) or ('/Auto_Other_Gases/' in f):
                data_year = f.split('.csv')[-2][-4:]
            elif ('/Auto_PM/' in f) or ('/Auto_PM10_Speciated/' in f) or ('/Auto_PM2.5_Speciated/' in f) or ('/Auto_PM1_Speciated/' in f) or ('/Auto_VOC/' in f):
                data_year = f.split('/')[-1][:4]
            elif '/Non-Auto/' in f:
                data_year = 'all'
            data_years.append(data_year)
        data_years = np.unique(data_years)

    elif 'US_EPA_AQS' in run_details['network']:
        for i, f in enumerate(data_files):
            data_years.append(f.split('_')[-1][:4])
        data_years = np.unique(data_years)
    
    elif 'US_EPA_CASTNET' in run_details['network']:
        for i, f in enumerate(data_files):
            if ('original_files/O3' in f) or ('original_files/hourly_gas' in f):
                data_years.append(f.split('_')[-1][:4])
            else:
                data_years.append('all')
        data_years = np.unique(data_years)

    return data_years

def get_YYYYMM_from_filenames(data_files, run_details):

    '''get unique list of data months as YYYYMM strings to process from data filenames.
       process for this differs by network.  
    ''' 

    data_months = []
    
    if 'AERONET' in run_details['network']:
        for i, f in enumerate(data_files):
            data_months.append(f.split('/')[-1][:6])
        data_months = np.unique(data_months)
    
    elif 'BJMEMC' in run_details['network']:
        for i, f in enumerate(data_files):
            data_months.append(f.split('/')[-1][:6])
        data_months = np.unique(data_months)

    elif 'CNEMC' in run_details['network']:
        for i, f in enumerate(data_files):
            data_months.append(f.split('/')[-1][:6])
        data_months = np.unique(data_months)
        
    return data_months

def get_YYYYMMDD_from_filenames(data_files, run_details):

    '''get unique list of data days as YYYYMMDD strings to process from data filenames.
       process for this differs by network.  
    ''' 

    data_days = []
    
    if 'AERONET' in run_details['network']:
        for i, f in enumerate(data_files):
            data_days.append(f.split('/')[-1].split('.html')[0])
        data_days = np.unique(data_days)
    
    if 'BJMEMC' in run_details['network']:
        for i, f in enumerate(data_files):
            data_days.append(f.split('/')[-1].split('.csv')[0])
        data_days = np.unique(data_days)

    if 'CNEMC' in run_details['network']:
        for i, f in enumerate(data_files):
            data_days.append(f.split('/')[-1].split('.csv')[0])
        data_days = np.unique(data_days)
    
    return data_days

def get_station_references_from_filenames(data_files, run_details):

    '''get unique list of raw station references provided in data filenames.
       process for this differs by network.
    '''

    station_references = []

    if run_details['network'] == 'CHILE_SINCA':
        for i, f in enumerate(data_files):
            station_references.append(f.split('/')[-2])  

    elif run_details['network'] == 'EANET':
        for i, f in enumerate(data_files):
            f = f.split('/')[-1]
            station_references.append(f.split('_')[0])

    elif run_details['network'] == 'EBAS':
        for i, f in enumerate(data_files):
            f = f.split('/')[-1]
            station_references.append(f.split('.')[0])

    elif 'EEA_AIRBASE' in run_details['network']:
        for i, f in enumerate(data_files):
            f_split = f.split('/')[-1]
            station_references.append(f_split[:7])

    elif 'EEA_AQ_eReporting' in run_details['network']:
        for i, f in enumerate(data_files):
            f_split = f.split('/')[-1]
            f_split = f_split.split('_')
            station_references.append('{}_{}_{}'.format(f_split[0],f_split[1],f_split[2]))

    elif 'NOAA_ISD' in run_details['network']:
        for i, f in enumerate(data_files):
            f_split = f.split('/')[-1].split('-')
            station_references.append('{}{}'.format(f_split[0], f_split[1]))

    elif 'US_EPA_AirNow_DOS' in run_details['network']:
        for i, f in enumerate(data_files):
            f_split = f.split('/')[-1]
            station_references.append(f_split.split('_')[0])

    elif run_details['network'] == 'US_NADP_AMNet':
        station_references.append('null')

    elif run_details['network'] == 'US_NADP_AMoN':
        station_references.append('null')

    elif run_details['network'] == 'WMO_WDCGG':
        for i, f in enumerate(data_files):
            f = f.split('/')[-1]
            station_references.append(f.split('_')[1])           

    #sort station references in a human-intuitive order
    station_references = sorted(remove_duplicates_preserve_order(station_references), key=natural_key)

    return station_references


def isUTF8(filename):
    data = open(filename, "rb").read()
    try:
        decoded = data.decode('UTF-8')
        return True
    except UnicodeDecodeError:
        return False

def predict_encoding(file_path, n_lines=100):

    '''Predict a file's encoding using chardet'''

    import chardet

    # Open the file as binary data
    with open(file_path, 'rb') as f:
        # Join binary lines for specified number of lines
        rawdata = b''.join([f.readline() for _ in range(n_lines)])

    return chardet.detect(rawdata)['encoding']


def determine_key_changing_metadata(old_metadata, new_metadata):

    '''function for determining if certain key metadata fields have changed between files
       returns True if metadata has changed, False if not
    '''

    #define key metadata fields to check
    key_metadata_fields = ['latitude', 'longitude', 'altitude', 'sampling_height', 'measurement_altitude', 'distance_to_building', 'distance_to_kerb', 'distance_to_junction', 'distance_to_source', 'street_width', 'street_type', 'daytime_traffic_speed', 'daily_passing_vehicles', 'ellipsoid', 'horizontal_datum', 'vertical_datum', 'projection', 'data_level', 'climatology', 'station_name', 'city', 'country', 'population', 'representative_radius', 'associated_networks',
                           'area_classification', 'station_classification', 'main_emission_source', 'land_use', 'terrain', 'measurement_scale',
                           'primary_sampling_type', 'primary_sampling_instrument_name', 'primary_sampling_instrument_reported_flow_rate',
                           'sample_preparation_types', 'sample_preparation_techniques', 
                           'measurement_methodology', 'measuring_instrument_name', 'measuring_instrument_sampling_type', 'measuring_instrument_reported_flow_rate', 'measuring_instrument_reported_lower_limit_of_detection', 'measuring_instrument_reported_upper_limit_of_detection', 'measuring_instrument_reported_uncertainty', 'measuring_instrument_reported_accuracy', 'measuring_instrument_reported_precision', 'measuring_instrument_reported_measurement_resolution', 'measuring_instrument_reported_absorption_cross_section', 'measuring_instrument_calibration_scale',
                           'network_provided_volume_standard_temperature','network_provided_volume_standard_pressure']

    #iterate through key metadata fields and check to see if there are differences between old/new metadata
    for key_metadata_field in key_metadata_fields:
        old_value = old_metadata[key_metadata_field]
        new_value = new_metadata[key_metadata_field]

        #if both values are NaN, continue to next field
        if (pd.isnull(old_value)) & (pd.isnull(new_value)):
            continue

        #test equivalence, if not equal, break out of function, as already know key metadata not equal between files
        if old_value != new_value:
            return True

    #if have made it through iteration of key metadata fields, key metadata is therefore equivalent
    return False


def redo_standardisations(dataframe, parameter_details, fields_to_standardise=None):

    '''take numpy dataframe for metadata/data and a list of fields that have been updated/modified ensure are standardised correctly.
       this is imperative for string formatting, e.g. to ensure capitalisations are correct, but the function will check standards of all passed fields.
    '''

    from GHOST_standards import get_standard_data, get_standard_metadata

    #get standard formats for dataframe
    if 'latitude' in dataframe.dtype.names:
        standard_dict = get_standard_metadata(parameter_details)
    else:
        standard_dict = get_standard_data(parameter_details)

    #define fields to standardise to be all fields if None defined
    if fields_to_standardise is None:
        fields_to_standardise = dataframe.dtype.names        

    #iterate through provided fields
    for key in fields_to_standardise:

        if key != 'measurement_qa_flags':

            #get list of values for field
            param_array = dataframe[key]

            #get standard data type for field
            standard_dtype = dataframe[key].dtype

            #get string format for field
            string_format = standard_dict[key]['string_format']

            #do string formatting for relevant fields before filling dataframe
            if (standard_dtype == np.dtype('O')) & (key not in ['flag', 'qa']):

                #upper case string formatting
                if 'upper' in string_format:
                    param_array = [np.str(x).upper() if pd.isnull(x) == False else x for x in param_array]

                #lower case string formatting
                elif 'lower' in string_format:
                    param_array = [np.str(x).lower() if pd.isnull(x) == False else x for x in param_array]

                #titalling string formatting
                elif 'title' in string_format:
                    param_array = [np.str(x).title() if pd.isnull(x) == False else x for x in param_array]

                #standard string formatting
                else:
                    param_array = [np.str(x) if pd.isnull(x) == False else x for x in param_array]

            #rewrite dataframe
            dataframe[key] = param_array

    return dataframe

def get_native_npz_shape(npz):

    '''get shape of parsed data arrays in native .npz station files
       does so in a lazy manner, without reading .npz file into memory'''

    import zipfile

    with zipfile.ZipFile(npz) as archive:
        npy = archive.open('data.npy')
        version = np.lib.format.read_magic(npy)
        shape, fortran, dtype = np.lib.format._read_array_header(npy, version)
        return shape[0]

def drop_stations(station_references, run_details, parameter_details):

    '''drop stations > than a particular max N stations number.
       preferentially keep the stations with the largest combined ranks of temporal representation and N measurements
    '''

    from scipy.stats import rankdata

    #if N station references is <= max_stations, then return
    if len(station_references) <= run_details['max_N_stations']:
        return station_references

    #get temporal representation and N parsed measurements for each station_reference
    temporal_representations = []
    N_measurements = []
    for station_reference in station_references:
        data_file = glob.glob(run_details['data_root']+'{}/{}/native/{}/{}*'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],station_reference))[0]
        data_file_split = data_file.split('_')
        start_time = int(data_file_split[-2])
        end_time = int(data_file_split[-1][:-4])
        temporal_representations.append(end_time - start_time)
        N_measurements.append(get_native_npz_shape(data_file))

    #get ranks of temporal representation and N measurements (largest = highest)
    temporal_representation_ranks = rankdata(temporal_representations)
    N_measurement_ranks = rankdata(N_measurements)
    #sum ranks 
    sum_ranks = temporal_representation_ranks + N_measurement_ranks

    #get indices of stations to retain (keeping stations associated with the highest ranks)
    station_inds_to_keep = (-sum_ranks).argsort()[:run_details['max_N_stations']]

    #return retained references
    return [station_references[ind] for ind in station_inds_to_keep] 


def list_duplicates(seq):

    '''Function for finding and returning values/indices of all duplicates in a string/list
       Returns list of tuples.
       The first value in each tuple is the duplicated value
       The second value in each tuple is a list of duplicated indices
       Returns empty list if no duplicates are found
       Function is optimised for speed
    '''

    from collections import defaultdict

    tally = defaultdict(list)
    for i,item in enumerate(seq):
        tally[item].append(i)
    return sorted(((key,locs) for key,locs in tally.items()
                            if len(locs)>1))


def remove_duplicates_preserve_order(seq):

    '''remove duplicates from list while preserving order'''

    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]


def merge_lists_sharing_common_elements(l):

    '''taking a list of lists merge all the lists that share common elements
       source: https://stackoverflow.com/questions/4842613/merge-lists-that-share-common-elements/4843408#4843408
    '''

    l = copy.deepcopy(l)
    len_l = len(l)
    i = 0
    while i < (len_l - 1):
        for j in range(i + 1, len_l):
            # i,j iterate over all pairs of l's elements including new
            # elements from merged pairs. We use len_l because len(l)
            # may change as we iterate
            i_set = set(l[i])
            j_set = set(l[j])
            if len(i_set.intersection(j_set)) > 0:
                # Remove these two from list
                l.pop(j)
                l.pop(i)
                # Merge them and append to the orig. list
                ij_union = list(i_set.union(j_set))
                l.append(ij_union)
                # len(l) has changed
                len_l -= 1
                # adjust 'i' because elements shifted
                i -= 1
                # abort inner loop, continue with next l[i]
                break
        i += 1
    return l


def natural_key(string_):

    '''key for sorting in human natural order'''

    return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)]


def read_from_numpy(numpy_file, desired_variables):

    '''read numpy .npz files, returning desired variables (input as list) stored in a dictionary: {desired_var_name: var}'''

    #first check if file is open in another process, if so wait for 10 seconds before trying to open again  
    lockfile = '{}/{}.lock'.format('/'.join(numpy_file.split('/')[:-1]),numpy_file.split('/')[-1].split('.npz')[0])
    while True:
        try:
            #create lockfile
            lf = os.open(lockfile, os.O_CREAT | os.O_EXCL)

            #load data
            numpy_dataframe = np.load(numpy_file, allow_pickle=True)

            loaded_variables = {}

            for desired_variable in desired_variables:
                loaded_variables[desired_variable] = numpy_dataframe[desired_variable]

            #close .npz file
            numpy_dataframe.close()

            #delete lockfile
            os.remove(lockfile)

            #break from while loop
            break

        except FileExistsError:
            #numpy file is currently being accessed by another process, wait another 5-15 seconds before trying again            
            sleep_secs = np.random.randint(low=5, high=16)
            time.sleep(sleep_secs)

    return loaded_variables

def cut_data_array(data_array,data_type,period_start,period_end):

    '''function which cuts data/metadata arrays by a given data range, provided as start minute from reference + end minute from reference'''

    if data_type == 'data':
        #cut relevant indices
        start_i = bisect.bisect_left(data_array['time'], period_start)
        end_i = bisect.bisect_left(data_array['time'], period_end)
 
        #if start_i == end_i, add 1 to end_i (to ensure cut is an array, not value)
        if start_i == end_i:
            end_i = end_i+1

        #cut data array
        data_array = data_array[start_i:end_i]

    if data_type == 'metadata':

        #cut relevant indices
        start_i = bisect.bisect_left(data_array['meta_update_stamp'], period_start)
        end_i = bisect.bisect_left(data_array['meta_update_stamp'], period_end)

        #flag for determining if need to update first metadata update stamp
        update_start_stamp = False

        #if data range starts and ends after last metadata update period, update to just take last metadata field
        if start_i == len(data_array['meta_update_stamp']):
            start_i = start_i - 1
            update_start_stamp = True

        #if have more than 1 metadata value and data range starts between metadata update periods, then adjust first metadata field to be from previous index
        if len(data_array['meta_update_stamp']) > 1: 
            if (period_start < data_array[start_i]['meta_update_stamp']) & (start_i != 0):
                start_i = start_i - 1
                update_start_stamp = True

        #if start_i == end_i, add 1 to end_i (to ensure cut is an array, not value)
        if start_i == end_i:
            end_i = end_i+1

        #cut metadata array
        data_array = data_array[start_i:end_i]

        #if need to update first metadata time stamp, do so now
        if update_start_stamp:
            data_array[0]['meta_update_stamp'] = period_start

    return data_array


def get_unique_processed_references(run_details, parameter_details):

    '''get unique processed station references from files saved out in the native/temporary file directories based on GHOST stage you are currently in'''

    processed_references = []

    #if current GHOST stage is in S stages, gather station files from (GHOST_stage-1)_temporary directory
    if 'S' in run_details['GHOST_stage']:
        previous_GHOST_stage = 'S{}'.format(int(run_details['GHOST_stage'][1:])-1)
        station_files = glob.glob(run_details['data_root']+'{}/{}/native/{}/temporary_{}/*.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],previous_GHOST_stage))
    #otherwise gather finalised station files from native directory
    else:
         station_files = glob.glob(run_details['data_root']+'{}/{}/native/{}/*.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name']))

    for file_i, file in enumerate(station_files):
        split = file.split('/')[-1].split('_')
        station_reference = '_'.join(split[:-2])
        processed_references.append(station_reference)

    #sort references in alphabetical order
    processed_references = sorted(np.unique(processed_references), key=natural_key)

    return processed_references


def get_data_from_reference(station_reference,run_details,parameter_details,read='all'):

    '''take station reference and return loaded metadata/data
       the type of data returned ('read') can be configured. By default both final stage metadata and data read and returned.
       read = 'all' --> read and return data + metadata (and filename); read = 'data' --> just read and return data (and filename); read = 'metadata' --> just read and return metadata (and filename)
    '''

    #if GHOST stage is in S stages, gather station filename from (GHOST_stage-1)_temporary directory
    if 'S' in run_details['GHOST_stage']:
        previous_GHOST_stage = 'S{}'.format(int(run_details['GHOST_stage'][1:])-1)
        station_numpy_filename = glob.glob(run_details['data_root']+'{}/{}/native/{}/temporary_{}/{}_*.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],previous_GHOST_stage,station_reference))[0]
     #otherwise gather filename from native directory
    else:
        station_numpy_filename = glob.glob(run_details['data_root']+'{}/{}/native/{}/{}_*.npz'.format(run_details['network'],run_details['ghost_version'],parameter_details['bsc_parameter_name'],station_reference))[0]

    #set desired variables to read
    if read == 'metadata':
        desired_variables = ['metadata']
    elif read == 'data':
        desired_variables = ['data']
    elif read == 'all':
        desired_variables = ['metadata', 'data']

    #read variables
    loaded_variables = read_from_numpy(station_numpy_filename, desired_variables)

    #return variables
    if read == 'metadata':
        return loaded_variables['metadata'], station_numpy_filename
    elif read == 'data':
        return loaded_variables['data'], station_numpy_filename
    elif read == 'all':
        return loaded_variables['metadata'], loaded_variables['data'], station_numpy_filename


def normalise_by_minmax(values, lower_limit=0.0, upper_limit=1.0, reverse=False):
    
    '''function that normalises list of values to lie within a given range (by default 0 to 1)
       the minimum value of the array will lie on the lower_limit,
       and the maximum_value of the array will lie on the upper_limit
       if reverse if set to be true, then the minimum value will lie on the upper limit,
       and the maximum value will lie on the lower limit.
    '''

    values = np.array(values, dtype=np.float32)
    uniques = np.unique(values)
    if len(uniques) == 0:
        return values
    elif len(uniques) == 1:
        return np.array([upper_limit]*len(values), dtype=np.float32)
    if reverse:
        values = values * -1.0
    width = upper_limit - lower_limit
    return (values - values.min())/(values.max() - values.min()) * width + lower_limit


def normalise_by_max(values, reverse=False, offset=0.0):
    
    '''function that normalises list of values by the maximum value in the array
       the maximum value will be normalised to be 1, and the rest as a ratio to the max
       if reverse is set to be True, then the ratio is made to the maximum after inverting the array
       if offset is non-zero, then the offset value is subsequently applied to all values.
       the input list of values input must be non-negative
    '''

    values = np.array(values, dtype=np.float32)
    uniques = np.unique(values)
    if len(uniques) == 0:
        return values
    elif len(uniques) == 1:
        return np.array([1.0+offset]*len(values), dtype=np.float32)

    if reverse:
        previous_min = np.min(values)
        values = values * -1.0
        values = values + (previous_min - (np.min(values)))
    return (values/np.max(values))+offset


def normalise_by_defined_max(values, max_val=None, offset=0.0):

    '''function that normalises list of values by a defined maximum value (if not set, by default will be max in data array)
       all values in the supplied data array will be set as a ratio to the max
       if offset is non-zero, then the offset value is subsequently applied to all values.
       the input list of values input must be non-negative
    '''

    values = np.array(values, dtype=np.float32)

    if max_val is None:
        max_val = np.max(values)   

    uniques = np.unique(values)
    if len(uniques) == 0:
        return values

    return (values/max_val)+offset


def return_PDF_intersection(pdf_1, pdf_2):
    
    '''function which returns intersection (between 0.0 and 1.0) between 2 PDFs with equivalent sample bases'''

    minima = np.minimum(pdf_1, pdf_2)
    intersection = np.true_divide(np.sum(minima), np.sum(pdf_2))
    return intersection


def filter_by_qa(qa_to_filter_by, data, qa, return_valid_mask=False):

    '''function that filters measurement data by a list of QA flags'''

    invalid_qa_flag_codes = [standard_QA_name_to_QA_code[invalid_flag_name] for invalid_flag_name in qa_to_filter_by]
    valid_mask = np.array([all(x not in qa_flag_codes for x in invalid_qa_flag_codes) for qa_flag_codes in qa])
    
    if return_valid_mask == False:
        return data[valid_mask]
    else:
        return valid_mask
    

def weighted_mode(list_of_values, weights):

    '''taking a list of values, and a list of weights associated with the values, calculate the weighted modal value
       this simply works by calculating the unique values in the list, then separately summing the weights associated with each unique value, and then determining which unique value is associated with the highest cumulative weights.
       if there is a tie, the lowest sorted value will be kept preferentially (i.e. a instead of b, 1 instead of 2)
       if there is only 1 unique value, then modal value is simply the only unique value
    '''

    #get unique values in list
    unique_values = np.unique(list_of_values)

    #if only 1 unique value, then modal value is simply the only unique value, return function
    if len(unique_values) == 1:
        return unique_values[0]

    #add all unique values to weight counter dictionary as keys, and give an initial value of 0.0
    weight_counter = {}
    for unique_value in unique_values:
        weight_counter[unique_value] = 0.0

    #iterate through all values in the list of values and associated weights
    for value, weight in zip(list_of_values,weights):
        #add weight associated with value to weight counter dictionary
        weight_counter[value] += weight

    #the unique value key associated the the greatest sum of weights in the weight counter dictionary is the modal value
    #if there is a tie, the lowest sorted key will be kept preferentially (i.e. a instead of b, 1 instead of 2)
    modal_value = max(weight_counter.keys(), key=(lambda key: weight_counter[key]))

    return modal_value


def get_relevant_yearmonths(run_details, parameter_details, temporal_resolution_code, region, stage_type='T'):

    '''create list of yearmonth strings, for which in those time periods exists valid processed data''' 

    from GHOST_standards import standard_parameters, standard_temporal_resolutions

    #get output temporal resolution specific variables
    temporal_resolution_vars = standard_temporal_resolutions[temporal_resolution_code]

    #set network directory where to read data from
    if region == 'GLOBAL':
        network_read_dir = run_details['network']
    else:
        network_read_dir = '{}_{}'.format(run_details['network'], region)

    #get relevant yearmonth of files for product type of stage
    #only keep months for which all parameters have months
    if stage_type == 'P':
        product_dict = get_default_product_parameters_per_network(run_details['network'])[run_details['parameter']]
        product_function_name = product_dict['function']
        calculation_parameters = product_dict['components']
        calculation_parameters_bsc = [standard_parameters[calculation_parameter]['bsc_parameter_name'] for calculation_parameter in calculation_parameters]
        max_calculation_parameters = len(calculation_parameters_bsc)
        if 'option' in product_dict:
            max_calculation_parameters = max_calculation_parameters - (len(product_dict['option']) - 1)

        all_yearmonth_strings = []
        for calculation_parameter_bsc in calculation_parameters_bsc:
            year_month_files = sorted(glob.glob(run_details['data_root'] + '{}/{}/{}/{}/{}_*.nc'.format(network_read_dir,run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],calculation_parameter_bsc,calculation_parameter_bsc)))
            if len(year_month_files) > 0:
                all_yearmonth_strings.extend([year_month_file.split('_')[-1][:6] for year_month_file in year_month_files])

        yearmonth_strings = []
        for name, count in zip(*np.unique(all_yearmonth_strings, return_counts=True)):
            if count == max_calculation_parameters:
                yearmonth_strings.append(name)

    #get relevant yearmonth of files for merge type of stage
    elif stage_type == 'M':
        networks = networks_to_merge(run_details['network'])
        all_yearmonth_strings = []
        for network in networks:
            year_month_files = sorted(glob.glob(run_details['data_root'] + '{}/{}/{}/{}/{}_*.nc'.format(network,run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],parameter_details['bsc_parameter_name'],parameter_details['bsc_parameter_name'])))
            if len(year_month_files) > 0:
                all_yearmonth_strings.extend([year_month_file.split('_')[-1][:6] for year_month_file in year_month_files])
        yearmonth_strings = np.unique(all_yearmonth_strings)

    # get relevant yearmonth of files for T stage
    elif stage_type == 'T':

        #create list for storing relevant yearmonth strings
        yearmonth_strings = []

        #create list of all years data has been processed for
        years = np.arange(int(run_details['start_year']),int(run_details['end_year'])+1)

        #iterate through years
        for year in years:

            #is there less than 1 year of data?
            if len(years) == 1:
                months = np.arange(int(run_details['start_month']), int(run_details['end_month']))

            #else, is it first process year?
            elif year == years[0]:
                months = np.arange(int(run_details['start_month']), 13)

            #else, is it last process year?
            elif year == years[-1]:
                months = np.arange(1,int(run_details['end_month']))

            #full year of months otherwise
            else:
                months = np.arange(1,13)

            #iterate through months
            for month in months:

                if month < 10:
                    month_string = '0'+str(month)
                    yearmonth_str = str(year)+month_string
                else:
                    month_string = str(month)
                    yearmonth_str = str(year)+month_string

                #check if have relevant files in year/month directory
                year_month_files = sorted(glob.glob(run_details['data_root'] + '{}/{}/{}/{}/temporary/{}/{}/*.nc'.format(network_read_dir,run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],parameter_details['bsc_parameter_name'],year,month_string)))

                #if so, then append yearmonth_str to list
                if len(year_month_files) > 0:
                    yearmonth_strings.append(yearmonth_str)

    return yearmonth_strings


def get_flag_simple(flag, qa):

    '''derive simplified flags (following WaterML2) from GHOST standardised flags''' 

    #create array for simplified flags
    flag_simple = np.empty((flag.shape[0]), dtype=np.uint8)

    #gather lists of relevant flags and qa
    unflagged_flag = standard_data_flag_name_to_data_flag_code['Un-Flagged Data']
    preliminary_flag = standard_data_flag_name_to_data_flag_code['Preliminary Data']
    missing_flag = standard_data_flag_name_to_data_flag_code['Missing Data']
    estimated_flag_names = ['Estimated Data - Unspecified','Estimated Data - Measured Negative Value',
                            'Estimated Data - No Value Detected','Estimated Data - Value Below Detection Limit',
                            'Estimated Data - Value Above Detection Limit',
                            'Estimated Data - Value Substituted from Secondary Monitor',
                            'Estimated Data - Multiple Parameters Aggregated']
    estimated_flags = np.array([standard_data_flag_name_to_data_flag_code[estimated_flag_name] for estimated_flag_name in estimated_flag_names], dtype=np.uint8)
    suspect_flag_names = ['Extreme/Irregular Data - Unspecified','High Variability of Data',
                          'No Value Detected','Reconstructed/Recalculated Data','Value Close to Detection Limit',
                          'Value Below Acceptable Range','Value Above Acceptable Range',
                          'Value Below Detection Limit','Value Above Detection Limit']
    suspect_flags = np.array([standard_data_flag_name_to_data_flag_code[suspect_flag_name] for suspect_flag_name in suspect_flag_names], dtype=np.uint8)

    ghost_invalid_qa = standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed']

    unchecked_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['unchecked']
    missing_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['missing']
    estimate_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['estimate']
    suspect_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['suspect']
    poor_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['poor']
    good_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['good']

    #loop through all times
    for time_ii, _ in enumerate(flag_simple):

        #1. unchecked data determined if have un-flagged data flag or preliminary data flag
        if (unflagged_flag in flag[time_ii,:]) or (preliminary_flag in qa[time_ii,:]):
            flag_simple[time_ii] = unchecked_simple_flag

        #2. missing data determined by missing measurement data flag
        elif missing_flag in qa[time_ii,:]:
            flag_simple[time_ii] = missing_simple_flag

        #3. estimate data determined by any of estimated data flags
        elif np.isin(flag[time_ii,:], estimated_flags).any():
            flag_simple[time_ii] = estimate_simple_flag

        #4. suspect data determined by any of suspect data flags + GHOST invalid QA
        elif (np.isin(flag[time_ii,:], suspect_flags).any()) & (ghost_invalid_qa in qa[time_ii,:]):
            flag_simple[time_ii] = suspect_simple_flag

        #5. poor data determined by GHOST invalid QA
        elif ghost_invalid_qa in qa[time_ii,:]:
            flag_simple[time_ii] = poor_simple_flag

        #6. otherwise data is considered good
        else:
            flag_simple[time_ii] = good_simple_flag  

    return flag_simple

def kde_fft(xin, gridsize=1024, extents=None, weights=None, adjust=1., bw='scott', xgrid=None):
    """
    A fft-based Gaussian kernel density estimate (KDE)
    for computing the KDE on a regular grid

    Note that this is a different use case than scipy's original
    scipy.stats.kde.gaussian_kde

    IMPLEMENTATION
    --------------

    Performs a gaussian kernel density estimate over a regular grid using a
    convolution of the gaussian kernel with a histogram of the data.

    It computes the sparse  histogram of the data samples where
    *x* is a 1-D sequence of the same length. If *weights* is None
    (default), this is a histogram of the number of occurences of the
    observations at (x[i]).
    histogram of the data is a faster implementation than numpy.histogram as it
    avoids intermediate copies and excessive memory usage!

    This function is typically *several orders of magnitude faster* than
    scipy.stats.kde.gaussian_kde.  For large (>1e7) numbers of points, it
    produces an essentially identical result.

    Boundary conditions on the data is corrected by using a symmetric /
    reflection condition. Hence the limits of the dataset does not affect the
    pdf estimate.

    INPUTS
    ------

        xin:  ndarray[ndim=1]
            The 1D data samples

        gridsize: int
            A nx integer of the size of the output grid (default: 1024)

        extents: (xmin, xmax) tuple
            tuple of the extents of output grid (default: extent of input data)

        weights: ndarray[ndim=1]
            An array of the same shape as x that weights each sample x_i
            by w_i.  Defaults to an array of ones the same size as x (default: None)

        adjust : float
            An adjustment factor for the bw. Bandwidth becomes bw * adjust.

        bw: str
            The method used to calculate the estimator bandwidth. 
            This can be 'scott' or 'silverman'(default: 'scott')

        xgrid: ndarray(ndim=1)
            The output grid (if this is provided gridsize and extents are ignored).

    OUTPUTS
    -------
        grid_points: ndarray[ndim=1]
            Grid points to sample density estimates

        kde: ndarray[ndim=1]
            A gridded 1D kernel density estimate of input data samples at grid points

    """
    # Variable check
    x = np.squeeze(np.asarray(xin))

    # Default extents are the extent of the data
    if xgrid is not None:
        xmin, xmax = xgrid.min(), xgrid.max()
        x = x[ (x <= xmax) & (x >= xmin) ]
    elif extents is None:
        xmin, xmax = x.min(), x.max()
    else:
        xmin, xmax = map(float, extents)
        x = x[ (x <= xmax) & (x >= xmin) ]

    n = x.size

    # apply weights
    if weights is None:
        # Default: Weight all points equally
        weights = np.ones(n)
    else:
        weights = np.squeeze(np.asarray(weights))
        if weights.size != x.size:
            raise ValueError('Input weights must be an array of the same size as xin!')

    # Optimize gridsize ------------------------------------------------------
    #Make grid and discretize the data and round it to the next power of 2
    # to optimize with the fft usage
    # ensure minimum gridsize of 1024 points
    if xgrid is None:
        if gridsize is None:
            gridsize = np.max((len(x), 1024.))
        gridsize = 2 ** np.ceil(np.log2(gridsize))  # round to next power of 2
        nx = int(gridsize)
    else:
        nx = len(xgrid)

    # Make the sparse histogram -------------------------------------------
    dx = (xmax - xmin) / (nx - 1)

    # Basically, this is just doing what np.digitize does with one less copy
    xyi = x - xmin
    xyi /= dx
    xyi = np.floor(xyi, xyi)
    xyi = np.vstack((xyi, np.zeros(n, dtype=int)))

    # Next, make a histogram of x
    # Exploit a sparse coo_matrix avoiding np.histogram due to excessive
    # memory usage with many points
    grid = coo_matrix((weights, xyi), shape=(nx, 1)).toarray()

    # Kernel Preliminary Calculations ---------------------------------------
    std_x = np.std(xyi[0])

    # Scaling factor for bandwidth
    if bw == 'scott': 
        bw_factor = (n ** (-1. / 5.)) * adjust
    elif bw == 'silverman':
        bw_factor =  ((n * 3 / 4.)**(-1. / 5)) * adjust 

    # Make the gaussian kernel ---------------------------------------------

    # First, determine the bandwidth using defined bandwidth estimator rule
    kern_nx = int(np.round(bw_factor * 2 * np.pi * std_x))

    # Then evaluate the gaussian function on the kernel grid
    kernel = np.reshape(gaussian(kern_nx, bw_factor * std_x), (kern_nx, 1))

    #---- Produce the kernel density estimate --------------------------------

    # Convolve the histogram with the gaussian kernel
    # use symmetric padding to correct for data boundaries in the kde
    npad = np.min((nx, 2 * kern_nx))
    grid = np.vstack( [grid[npad: 0: -1], grid, grid[nx: nx - npad: -1]] )
    grid = convolve(grid, kernel, mode='same')[npad: npad + nx]

    # Normalization factor to divide result by so that units are in the same
    # units as scipy.stats.kde.gaussian_kde's output.
    norm_factor = 2 * np.pi * std_x * std_x * bw_factor ** 2
    norm_factor = n * dx * np.sqrt(norm_factor)

    # Normalize the result
    grid /= norm_factor

    # return grid points and estimated densities 
    if xgrid is None:
        return np.linspace(xmin,xmax,nx), np.squeeze(grid)
    else:
        return np.squeeze(grid)