#WRITTEN BY DENE BOWDALO

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###

#raw_to_npy

#take network provided external metadata of whatever form and process to numpy .npy format for efficient reading
#each network has different types of reported metadata and therefore different requirements

###--------------------------------------------------------------------------------------------------###
###IMPORT MODULES
###--------------------------------------------------------------------------------------------------###

import copy
import csv
import glob
import os
import xml.etree.ElementTree as ET

import numpy as np

from GHOST_standards import standard_networks

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###

#Define GHOST version
ghost_version = '1.5.1'

#Define list of networks' metadata to process
#networks_to_process = ['BJMEMC','CANADA_NAPS','CHILE_SINCA','CNEMC','EBAS','EEA_AIRBASE','EEA_AQ_eReporting','JAPAN_NIES','MEXICO_CDMX','MITECO','NOAA_ISD','UK_AIR','US_EPA_AirNow_DOS','US_EPA_AQS','US_EPA_CASTNET','US_NADP_AMNet','US_NADP_AMoN']
networks_to_process = ['EBAS']

#Define root data directory (varies by machine)
#try:
#    machine = os.environ['BSC_MACHINE']
#except:
#    machine = ''
##power9/mn4/mn5
#if (machine == 'power') or (machine == 'mn4') or (machine == 'mn5'):
#    data_root = '/gpfs/projects/bsc32/AC_cache/obs/ghost/'
##workstation
#else:
#    data_root = '/esarchive/obs/ghost/'
#data_root = '/esarchive/obs/ghost/'
data_root = '/gpfs/projects/bsc32/AC_cache/obs/ghost/'

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###

#define functions used for transformation of raw external metadata to npy stored data

###------------------------

def get_raw_metadata_file_location(filename, network):
    
    '''Return location of a desired external metadata file''' 

    return data_root+network+'/metadata/'+ghost_version+'/network_provided/'+filename


def get_npy_metadata_file_location(filename, network):

    '''Return location to store new .npy external metadata file'''

    return data_root+network+'/metadata/'+ghost_version+'/processed/'+filename


###------------------------

def csv_to_npy(external_metadata_file, numpy_filename, delimiter=',', quotechar='', encoding='utf-8-sig'):

    '''Define function to read a standard CSV file and convert to .npy''' 

    print(external_metadata_file)
    with open(external_metadata_file, 'r', encoding=encoding) as f:
        if quotechar == '':
            reader = csv.reader(f,delimiter=delimiter)
        else:
            reader = csv.reader(f,delimiter=delimiter,quotechar=quotechar)
        for row_i, row in enumerate(reader):
            if row_i == 0:
                original_meta_variable_names = copy.deepcopy(row)
                #do not process 'Null' or empty fields
                meta_variable_names = [name for name in original_meta_variable_names if name not in ['Null','']]   
                external_metadata = [[] for _ in range(len(meta_variable_names))]
            else:
                if (len(row) != len(original_meta_variable_names)) & (row_i == 1):
                    print(len(row), len(original_meta_variable_names))
                for meta_variable_name in meta_variable_names:
                    external_metadata[meta_variable_names.index(meta_variable_name)].append(row[original_meta_variable_names.index(meta_variable_name)].strip())
            
        #convert to numpy record array, to allow indexing with column names
        external_metadata = np.rec.fromarrays(external_metadata, names=meta_variable_names)

        #save as .npy file
        np.save(numpy_filename, external_metadata)

###------------------------

def EEA_AIRBASE_xml_to_npy(numpy_filename):

    '''Convert EEA AIRBASE xml metadata to .npy'''

    #read xml station data
    xml_dir = data_root+'/EEA_AIRBASE/original_files/AirBase_v8_xmldata'

    #root of specific station and measurement configuration information
    station_file = data_root+'/EEA_AIRBASE/original_files/AirBase_v8_stations.csv' 
    measurement_file = data_root+'/EEA_AIRBASE/original_files/AirBase_v8_measurement_configurations.csv'

    #get all xml files
    xml_files = sorted(glob.glob('%s/*.xml'%(xml_dir)))

    tags = []

    #define variable name, and path to variable

    country_map = {'country_eu_member':'country_eu_member',
                   'country_iso_code' :'country_iso_code',
                   'country_name'     :'country_name'}

    network_map = {'network_name':                                      'network_name', 
                   'network_short_name':                                'network_short_name',
                   'network_type':                                      'network_type',
                   'network_start_date':                                'network_start_date',
                   'network_end_date':                                  'network_end_date',
                   'network_time_reference_basis':                      'network_time_reference_basis',
                   'network_data_supplier_organization_address':        'network_data_supplier/organization/organization_address',
                   'network_data_supplier_organization_city':           'network_data_supplier/organization/organization_city',
                   'network_data_supplier_organization_fax_number':     'network_data_supplier/organization/organization_fax_number',
                   'network_data_supplier_organization_name':           'network_data_supplier/organization/organization_name',
                   'network_data_supplier_organization_phone_number':   'network_data_supplier/organization/organization_phone_number',
                   'network_data_supplier_organization_website_address':'network_data_supplier/organization/organization_website_address',
                   'network_data_supplier_person_email_address':        'network_data_supplier/person/person_email_address',
                   'network_data_supplier_person_first_name':           'network_data_supplier/person/person_first_name',
                   'network_data_supplier_person_last_name':            'network_data_supplier/person/person_last_name',
                   'network_data_supplier_person_title':                'network_data_supplier/person/person_title',
                   'network_manager_organization_address':              'network_manager/organization/organization_address',
                   'network_manager_organization_city':                 'network_manager/organization/organization_city',
                   'network_manager_organization_fax_number':           'network_manager/organization/organization_fax_number',
                   'network_manager_organization_name':                 'network_manager/organization/organization_name',
                   'network_manager_organization_phone_number':         'network_manager/organization/organization_phone_number',
                   'network_manager_organization_website_address':      'network_manager/organization/organization_website_address',
                   'network_manager_person_email_address':              'network_manager/person/person_email_address',
                   'network_manager_person_first_name':                 'network_manager/person/person_first_name',
                   'network_manager_person_last_name':                  'network_manager/person/person_last_name',
                   'network_manager_person_title':                      'network_manager/person/person_title'}

    station_map_1 = {'network_code':                        'network_info/network_code',
                     'EMEP_station':                        'station_info/EMEP_station',
                     'lau_level1_code':                     'station_info/lau_level1_code',
                     'lau_level2_code':                     'station_info/lau_level2_code',
                     'lau_level2_name':                     'station_info/lau_level2_name',
                     'main_emission_source':                'station_info/main_emission_source',
                     'meteorological_parameter':            'station_info/meteorological_parameter',
                     'monitoring_obj':                      'station_info/monitoring_obj',
                     'monitoring_obj_other_text':           'station_info/monitoring_obj_other_text',
                     'number_of_vehicles':                  'station_info/number_of_vehicles',
                     'population':                          'station_info/population',
                     'sabe_country_code':                   'station_info/sabe_country_code',
                     'sabe_unit_code':                      'station_info/sabe_unit_code',
                     'sabe_unit_name':                      'station_info/sabe_unit_name',
                     'station_airbase_code':                'station_info/station_airbase_code',
                     'station_altitude':                    'station_info/station_altitude',
                     'station_characteristic_of_zone':      'station_info/station_characteristic_of_zone',
                     'station_city':                        'station_info/station_city',
                     'station_description':                 'station_info/station_description',
                     'station_distance_from_kerb':          'station_info/station_distance_from_kerb',
                     'station_distance_to_junction':        'station_info/station_distance_to_junction',
                     'station_distance_to_source':          'station_info/station_distance_to_source',
                     'station_end_date':                    'station_info/station_end_date',
                     'station_latitude_decimal_degrees':    'station_info/station_latitude_decimal_degrees',
                     'station_latitude_dms':                'station_info/station_latitude_dms',
                     'station_local_code':                  'station_info/station_local_code',
                     'station_longitude_decimal_degrees':   'station_info/station_longitude_decimal_degrees',
                     'station_longitude_dms':               'station_info/station_longitude_dms',
                     'station_lorry_percentage':            'station_info/station_lorry_percentage',
                     'station_name':                        'station_info/station_name',
                     'station_nuts_level0':                 'station_info/station_nuts_level0',
                     'station_nuts_level1':                 'station_info/station_nuts_level1',
                     'station_nuts_level2':                 'station_info/station_nuts_level2',
                     'station_nuts_level3':                 'station_info/station_nuts_level3',
                     'station_ozone_classification':        'station_info/station_ozone_classification',
                     'station_ratio_distance':              'station_info/station_ratio_distance',
                     'station_represented_length_of_street':'station_info/station_represented_length_of_street',
                     'station_represented_radius':          'station_info/station_represented_radius',
                     'station_start_date':                  'station_info/station_start_date',
                     'station_street_width':                'station_info/station_street_width',
                     'station_subcategory_rural_background':'station_info/station_subcategory_rural_background',
                     'station_traffic_speed':               'station_info/station_traffic_speed',
                     'station_type_of_area':                'station_info/station_type_of_area',
                     'street_name':                         'station_info/street_name',
                     'street_type':                         'station_info/street_type',
                     'type_of_station':                     'station_info/type_of_station'}                 

 
    station_map_2 = {'component_FWD':                               'component_FWD',
                     'component_caption':                           'component_caption',
                     'component_code':                              'component_code',
                     'component_name':                              'component_name',
                     'measurement_unit':                            'measurement_unit',
                     'measurement_european_group_code':             'measurement_european_group_code',
                     'measurement_group_start_date':                'measurement_group_start_date',
                     'measurement_group_end_date':                  'measurement_group_end_date',
                     'measurement_latest_date_available_in_AIRBASE':'measurement_latest_date_available_in_AIRBASE'}
 
    station_map_3 = {'body_or_programme':              'body_or_programme',
                     'calibration_frequency':          'calibration_frequency',
                     'calibration_method':             'calibration_method',
                     'calibration_unit':               'calibration_unit',
                     'height_sampling_point':          'height_sampling_point',
                     'integration_time_frequency':     'integration_time_frequency',
                     'integration_time_unit':          'integration_time_unit',
                     'length_sampling_line':           'length_sampling_line',
                     'location_sampling_point':        'location_sampling_point',
                     'measurement_automatic':          'measurement_automatic',
                     'measurement_start_date':         'measurement_start_date',
                     'measurement_end_date':           'measurement_end_date',
                     'measurement_equipment':          'measurement_equipment',
                     'measurement_european_code':      'measurement_european_code',
                     'measurement_technique_principle':'measurement_technique_principle',
                     'sampling_time':                  'sampling_time',
                     'sampling_time_unit':             'sampling_time_unit'}

    all_ordered_vars = ['station_european_code']
    for var_list in [list(country_map.keys()), list(network_map.keys()), list(station_map_1.keys()), list(station_map_2.keys()), list(station_map_3.keys())]:
        all_ordered_vars.extend(var_list)

    #read station specific file
    station_data_load = np.loadtxt(station_file,dtype=object,delimiter='\t')
    station_data_columns = copy.deepcopy(station_data_load[0,:])
    station_data_extra_vars = []
    #change some field names to be as they are named in AIRBASE xml files
    if 'station_subcat_rural_back' in station_data_columns:
        station_data_columns[np.where(station_data_columns == 'station_subcat_rural_back')[0]] = 'station_subcategory_rural_background'
    if 'station_longitude_deg' in station_data_columns:
        station_data_columns[np.where(station_data_columns == 'station_longitude_deg')[0]] = 'station_longitude_decimal_degrees'
    if 'station_latitude_deg' in station_data_columns:
        station_data_columns[np.where(station_data_columns == 'station_latitude_deg')[0]] = 'station_latitude_decimal_degrees'
    for var in station_data_columns:
        if var not in all_ordered_vars:
            all_ordered_vars.append(var)
            station_data_extra_vars.append(var)
    
    #read measurement configuration specific file
    measurement_data_load = np.loadtxt(measurement_file,dtype=object,delimiter='\t')    
    measurement_data_columns = copy.deepcopy(measurement_data_load[0,:])
    measurement_data_extra_vars = []
    for var in measurement_data_columns:
        if var not in all_ordered_vars:
            all_ordered_vars.append(var)
            measurement_data_extra_vars.append(var)

    all_data = []
    #iterate through each file
    for ii, xml_file in enumerate(xml_files):
        root = ET.parse(xml_file).getroot()
    
        country_dict = {}
        network_dict = {}
        station_dict = {}
    
        #process country information
        find_result = root.findall('country')
        for result in find_result:
            for country_var in list(country_map.keys()):        
                var_find = result.find(country_map[country_var])
                if var_find is not None: 
                    country_dict[country_var] = var_find.text

        #process network information
        find_result = root.findall('country/network')
        for result in find_result:
            current_network_code = result.find('network_code').text
            network_dict[current_network_code] = {}
            for network_var in list(network_map.keys()):    
                var_find = result.find(network_map[network_var])  
                if var_find is not None:  
                     network_dict[current_network_code][network_var] = var_find.text
        
        #process station information
        find_result = root.findall('country/station')
        for result in find_result:
            current_station_code = result.find('station_european_code').text
            station_dict[current_station_code] = {}
            for station_var in list(station_map_1.keys()):
                var_find = result.find(station_map_1[station_var])
                if var_find is not None:
                    station_dict[current_station_code][station_var] = var_find.text
            find_result_2 = result.findall('measurement_configuration')
            for result_2_code, result_2 in enumerate(find_result_2):
                result_2_code = str(result_2_code)
                station_dict[current_station_code][result_2_code] = {}
                for station_var in list(station_map_2.keys()):
                    var_find = result_2.find(station_map_2[station_var])
                    if var_find is not None:
                        station_dict[current_station_code][result_2_code][station_var] = var_find.text
                find_result_3 = result_2.findall('measurement_info')
                for result_3_code, result_3 in enumerate(find_result_3):
                    result_3_code = str(result_3_code)
                    station_dict[current_station_code][result_2_code][result_3_code] = {}
                    for station_var in list(station_map_3.keys()):
                        var_find = result_3.find(station_map_3[station_var])
                        if var_find is not None:
                            station_dict[current_station_code][result_2_code][result_3_code][station_var] = var_find.text
    
        #process data into format for saving as .npy
    
        #there should be a unique line for station, by measurement configuration, by measurement info,
        for station_code in list(station_dict.keys()):
            print(station_code)       
 
            specific_station_dict = station_dict[station_code]
            station_data = []
        
            #make join between station_code and station data loaded
            specific_station_data_load = station_data_load[(station_data_load[:,np.where(station_data_columns == 'station_european_code')[0]] == station_code).flatten(),:]
            if specific_station_data_load.shape[0] != 1:
                print('CANNOT MAKE SPECIFIC STATION DATA LOAD JOIN')
                print(station_code, specific_station_data_load)
                print(specific_station_data_load.shape)
                1+'a'

            #first variable is station_code
            station_data.append(station_code)
            #pass in country information
            for country_var in list(country_map.keys()):
                if country_var in list(country_dict.keys()):
                    station_data.append(country_dict[country_var])
                else:
                    station_data.append('')
           
            #pass in network information (take network_code from station_dict)
            for network_var in list(network_map.keys()):
                if network_var in list(network_dict[specific_station_dict['network_code']].keys()):
                    station_data.append(network_dict[specific_station_dict['network_code']][network_var])
                else:
                    station_data.append('') 
 
            #pass in station information
            for station1_var in list(station_map_1.keys()):
                if station1_var in list(specific_station_dict.keys()):
                    station_data.append(specific_station_dict[station1_var])
                else:
                    station_data.append('')
                
            config_vars = [var for var in list(specific_station_dict.keys()) if var not in list(station_map_1.keys())]
            for config_var in config_vars:
                station_data_config = copy.deepcopy(station_data)
                specific_config_dict = specific_station_dict[config_var]
                for station2_var in list(station_map_2.keys()):
                    if station2_var in list(specific_config_dict.keys()):
                        station_data_config.append(specific_config_dict[station2_var])
                    else:
                        station_data_config.append('')
                    if station2_var == 'component_code':
                        current_component_code = copy.deepcopy(station_data_config[-1])
                    if station2_var == 'measurement_european_group_code':
                        current_measurement_group_code = copy.deepcopy(station_data_config[-1])

                measurement_info_vars = [var for var in list(specific_config_dict.keys()) if var not in list(station_map_2.keys())]
                for measurement_info_var in measurement_info_vars:
                    station_data_measurement_info = copy.deepcopy(station_data_config)
                    specific_measurement_info_dict = specific_config_dict[measurement_info_var]
                    for station3_var in list(station_map_3.keys()):
                        if station3_var in list(specific_measurement_info_dict.keys()):
                            station_data_measurement_info.append(specific_measurement_info_dict[station3_var])
                        else:
                            station_data_measurement_info.append('')
                        if station3_var == 'measurement_european_code':
                            current_measurement_code = copy.deepcopy(station_data_measurement_info[-1])

                            #make join between station_code:component_code:measurement_european_code and measurement data loaded
                            specific_measurement_data_load = measurement_data_load[((measurement_data_load[:,np.where(measurement_data_columns == 'station_european_code')[0]] == station_code).flatten()) & ((measurement_data_load[:,np.where(measurement_data_columns == 'component_code')[0]] == current_component_code).flatten()) & ((measurement_data_load[:,np.where(measurement_data_columns == 'measurement_european_group_code')[0]] == current_measurement_group_code).flatten()) & ((measurement_data_load[:,np.where(measurement_data_columns == 'measurement_european_code')[0]] == current_measurement_code).flatten()),:]
                            if specific_measurement_data_load.shape[0] != 1:
                                print('CANNOT MAKE SPECIFIC MEASUREMENT DATA LOAD JOIN')
                                print(station_code, current_component_code, current_measurement_group_code, current_measurement_code, specific_measurement_data_load)
                                print(specific_measurement_data_load.shape)
                                1+'a'
                
                    #add station specific load file data
                    for var_ii, var in enumerate(station_data_columns):
                        #check if pre-written var or not
                        if var not in station_data_extra_vars:
                            #overwrite field only if field is currently empty
                            if station_data_measurement_info[all_ordered_vars.index(var)] == '':
                                if specific_station_data_load[0,var_ii] != '':
                                    print(var,specific_station_data_load[0,var_ii])
                                    station_data_measurement_info[all_ordered_vars.index(var)] = specific_station_data_load[0,var_ii]
                        #if not-existing, then append
                        else:
                            station_data_measurement_info.append(specific_station_data_load[0,var_ii])

                    #add measurement specific load file data
                    for var_ii, var in enumerate(measurement_data_columns):
                        #check if pre-written var or not
                        if var not in measurement_data_extra_vars:
                            #overwrite field only if field is currently empty
                            if station_data_measurement_info[all_ordered_vars.index(var)] == '':
                                if specific_measurement_data_load[0,var_ii] != '':
                                    print(var,specific_measurement_data_load[0,var_ii])
                                    station_data_measurement_info[all_ordered_vars.index(var)] = specific_measurement_data_load[0,var_ii]
                        #if not-existing, then append
                        else:
                            station_data_measurement_info.append(specific_measurement_data_load[0,var_ii])

                    #convert full station specific data to tuple
                    all_data.append(tuple(station_data_measurement_info))
    
    #create lists for standardising data formats of the input data
    standard_dtype = [(var_name,object) for var_name in all_ordered_vars]

    #create numpy dataframe to store standardised forms of variables inside data dictionary
    dataframe = np.array(all_data, dtype=standard_dtype)

    #save out to .npy file
    np.save(numpy_filename, dataframe)

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###

#define external metadata files to convert to numpy, by network                 
if __name__ == "__main__":

    #iterate through defined networks to process
    for network in networks_to_process:

        #for BJMEMC, meta file per YYYYMMDD, so loop through and process all
        if network == 'BJMEMC':
            #get location of raw primary metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['BJMEMC']['external_metadata_files'][0], 'BJMEMC')
            #get all data years to process 
            yearly_files = sorted(glob.glob('{}*'.format(raw_meta_file_location.split('YYYYMMDD')[0])))
            #iterate through and process all annual files
            for yearly_file in yearly_files:
                yearly_file_cut = yearly_file.split('/')[-1]
                yearly_file_npy = yearly_file_cut.replace('.csv','.npy')
                #get location to store new .npy external metadata file
                npy_meta_file_location = get_npy_metadata_file_location(yearly_file_npy, 'BJMEMC')          
                #convert csv file to .npy
                csv_to_npy(yearly_file, npy_meta_file_location,delimiter=';')

        elif network == 'CANADA_NAPS':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['CANADA_NAPS']['external_metadata_files'][0], 'CANADA_NAPS')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('CANADA_NAPS_META.npy', 'CANADA_NAPS')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location,delimiter=',',quotechar='"',encoding='ISO-8859-1')

        elif network == 'CAPMoN':
            pass

        elif network == 'CHILE_SINCA':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['CHILE_SINCA']['external_metadata_files'][0], 'CHILE_SINCA')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('CHILE_SINCA_META.npy', 'CHILE_SINCA')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location)

        #for CNEMC, meta file per year, so loop through and process all
        elif network == 'CNEMC':
            #get location of raw primary metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['CNEMC']['external_metadata_files'][0], 'CNEMC')
            #get all data years to process 
            yearly_files = sorted(glob.glob('{}*'.format(raw_meta_file_location.split('YYYY')[0])))
            #iterate through and process all annual files
            for yearly_file in yearly_files:
                yearly_file_cut = yearly_file.split('/')[-1]
                yearly_file_npy = yearly_file_cut.replace('.csv','.npy')
                #get location to store new .npy external metadata file
                npy_meta_file_location = get_npy_metadata_file_location(yearly_file_npy, 'CNEMC')          
                #convert csv file to .npy
                csv_to_npy(yearly_file, npy_meta_file_location)

        #EBAS external metadata stems from a manually compiled .txt file, with a line per station
        elif network == 'EBAS':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['EBAS']['external_metadata_files'][0], 'EBAS')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('EBAS_META.npy', 'EBAS')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location)

        #EEA AIRBASE provides metadata in multiple xml files
        elif network == 'EEA_AIRBASE':
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('EEA_AIRBASE_META.npy', 'EEA_AIRBASE') 
            #convert .xml metadata to .npy
            EEA_AIRBASE_xml_to_npy(npy_meta_file_location)

        #EEA AQ eReporting provide 2 separate external metadata files, which carry on almost all of the conventions from EEA AIRBASE
        elif network == 'EEA_AQ_eReporting':
            #get location of raw primary metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['EEA_AQ_eReporting']['external_metadata_files'][0], 'EEA_AQ_eReporting')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('EEA_AQ_eReporting_META.npy', 'EEA_AQ_eReporting')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location, delimiter='\t')

            #get location of raw secondary AIDE D metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['EEA_AQ_eReporting']['external_metadata_files'][1], 'EEA_AQ_eReporting')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('EEA_AQ_eReporting_AIDE_D_META.npy', 'EEA_AQ_eReporting')          
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location)

        #for JAPAN, meta file per year, so loop through and process all
        elif network == 'JAPAN_NIES':
            #get location of raw primary metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['JAPAN_NIES']['external_metadata_files'][0], 'JAPAN_NIES')
            #get all data years to process 
            yearly_files = sorted(glob.glob('{}*'.format(raw_meta_file_location.split('YYYY')[0])))
            #iterate through and process all annual files
            for yearly_file in yearly_files:
                yearly_file_cut = yearly_file.split('/')[-1]
                yearly_file_npy = yearly_file_cut.replace('.txt','.npy')
                #get location to store new .npy external metadata file
                npy_meta_file_location = get_npy_metadata_file_location(yearly_file_npy, 'JAPAN_NIES')          
                #convert csv file to .npy
                csv_to_npy(yearly_file, npy_meta_file_location, encoding='cp932')
            
        elif network == 'MEXICO_CDMX':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['MEXICO_CDMX']['external_metadata_files'][0], 'MEXICO_CDMX')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('MEXICO_CDMX_META.npy', 'MEXICO_CDMX')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location, quotechar='"')

        elif network == 'MITECO':
            #get raw primary metadata file root
            raw_meta_file_root = get_raw_metadata_file_location('MITECO_META', 'MITECO')
            #get location of all annual metadata files
            raw_meta_file_locations = sorted(glob.glob('{}*.csv'.format(raw_meta_file_root)))
            #iterate through raw_meta_file_locations
            for raw_meta_file_location in raw_meta_file_locations:
                #get metadata file year
                year = raw_meta_file_location.split('_')[-1][:4]
                #get location to store new .npy external metadata file
                npy_meta_file_location = get_npy_metadata_file_location('MITECO_META_{}.npy'.format(year), 'MITECO')
                #convert csv file to .npy
                csv_to_npy(raw_meta_file_location, npy_meta_file_location, delimiter=';')

        elif network == 'NOAA_ISD':
            # get location of raw primary metadata file
            raw_meta_file_location = get_raw_metadata_file_location(
                standard_networks['NOAA_ISD']['external_metadata_files'][0], 'NOAA_ISD')
            # get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('NOAA_ISD.npy', 'NOAA_ISD')
            # convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location)

        elif network == 'SEARCH':
            pass

        elif network == 'UK_AIR':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['UK_AIR']['external_metadata_files'][0], 'UK_AIR')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('UK_AIR_META.npy', 'UK_AIR')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location, delimiter=',', quotechar='"')

        elif network == 'US_EPA_AirNow_DOS':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['US_EPA_AirNow_DOS']['external_metadata_files'][0], 'US_EPA_AirNow_DOS')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('US_EPA_AirNow_DOS_META.npy', 'US_EPA_AirNow_DOS')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location, delimiter=',')

        elif network == 'US_EPA_AQS':
            #get location of raw primary metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['US_EPA_AQS']['external_metadata_files'][0], 'US_EPA_AQS')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('US_EPA_AQS_MONITORS.npy', 'US_EPA_AQS')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location, delimiter=',', quotechar='"')

            #get location of raw primary metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['US_EPA_AQS']['external_metadata_files'][1], 'US_EPA_AQS')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('US_EPA_AQS_SITES.npy', 'US_EPA_AQS')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location, delimiter=',',quotechar='"')

        elif network == 'US_EPA_CASTNET':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['US_EPA_CASTNET']['external_metadata_files'][0], 'US_EPA_CASTNET')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('US_EPA_CASTNET_META.npy', 'US_EPA_CASTNET')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location, quotechar='"')

        elif network == 'US_NADP_AMNet':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['US_NADP_AMNet']['external_metadata_files'][0], 'US_NADP_AMNet')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('US_NADP_AMNet_META.npy', 'US_NADP_AMNet')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location,delimiter=',',quotechar='"')

        elif network == 'US_NADP_AMoN':
            #get location of raw external metadata file
            raw_meta_file_location = get_raw_metadata_file_location(standard_networks['US_NADP_AMoN']['external_metadata_files'][0], 'US_NADP_AMoN')
            #get location to store new .npy external metadata file
            npy_meta_file_location = get_npy_metadata_file_location('US_NADP_AMoN_META.npy', 'US_NADP_AMoN')
            #convert csv file to .npy
            csv_to_npy(raw_meta_file_location, npy_meta_file_location,delimiter=',',quotechar='"')
