import copy
import calendar
import datetime
import glob
import os
import time

import cartopy.crs as ccrs
import geopandas as gpd
from netCDF4 import Dataset
import numpy as np
import pandas as pd
from scipy import stats
from shapely.geometry import MultiPolygon, Polygon, Point, shape

from crossnetwork_processing_functions import get_data_from_reference, weighted_mode, redo_standardisations 
from geographic_functions import get_unique_longitude_latitude_pairs, local_timezone_name, reverse_geocode_station_information, get_coordinate_gridcell_indices, get_neighbouring_gridcell_indices, calculate_destination_points_perimeter, get_relevant_gridbox_indices
from time_functions import datetime_to_fractional_minutes_from_reference


class GloballyGriddedMetadata:

    '''Class that sets globally gridded metadata variables per station.''' 

    def __init__(self, station_reference, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- GLOBALLY GRIDDED METADATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #get metadata/data for reference
        self.metadata, self.data, self.station_numpy_filename = get_data_from_reference(self.station_reference,self.run_details,self.parameter_details,read='all')

        #define list to append all names of metadata fields which are updated
        self.metadata_fields_to_update = []

        #get unique longitude and latitude pairs across metadata, with associated metadata/data indices of each pair
        self.unique_pair_dictionary = get_unique_longitude_latitude_pairs(self.metadata, self.data)

        #For all unique longitude/latitude coordinate pairs, calculate perimeter coordinates 5km/25km in distance from initial coordinates, around 360 degrees (calculated geodesically)
        perimeter_radii = [5.0, 25.0]
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():
            for perimeter_radius in perimeter_radii:
                perimeter_points, perimeter_extreme_points = calculate_destination_points_perimeter(unique_pair['longitude'],unique_pair['latitude'],perimeter_radius)
                self.unique_pair_dictionary[unique_pair_ii]['points_{}'.format(perimeter_radius)] = perimeter_points        
                self.unique_pair_dictionary[unique_pair_ii]['extreme_{}'.format(perimeter_radius)] = perimeter_extreme_points  

        #get UMBC Anthrome classifications
        self.get_UMBC_Anthrome()

        #get ASTER altitude
        self.get_ASTER()

        #get ETOPO1 altitude
        self.get_ETOPO1()

        #get GSFC coastline proximity
        self.get_GSFC_coastline_proximity()

        #get Koppen Geiger classification
        self.get_Koppen_Geiger_classification()

        #get Meybeck Classification
        self.get_Meybeck_classification()

        #get Iwahashi Classification
        self.get_Iwahashi_classification()

        #get MODIS classifications
        self.get_MODIS_classifications()

        #get GHSL area/population densities and settlement model classifications
        self.get_GHSL()

        #get NOAA DMSP OLS nighttime lights
        self.get_NOAA_DMSP_OLS()

        #get GPW population density
        self.get_GPW_population_density()

        #get OMI NO2 annual column averages
        self.get_OMI_NO2()

        #get EDGAR emissions data
        self.get_EDGAR_emissions()

        #get Joly-Peuch classification codes
        self.get_Joly_Peuch_classification_codes()

        #get WWF TEOW classifications
        self.get_WWF_TEOW_classifications()

        # get NUTS regions
        self.get_NUTS_region()

        #get WMO region
        self.get_WMO_region()

        #get reverse geocode staton information 
        self.get_reverse_geocode_station_information()

        #get local station timezone information
        self.get_station_local_timezone()

        #get WIGOS station identifier
        self.get_WIGOS_station_identifier()

        #update standardisations for modified metadata fields
        self.metadata = redo_standardisations(self.metadata, self.parameter_details, self.metadata_fields_to_update)


    def get_UMBC_Anthrome(self):

        '''parse globally gridded UMBC anthrome classifications'''

        self.run_details['log_file_str'] += 'GETTING UMBC ANTHROME METADATA\n'

        #define all distinct Anthrome classifications by code:string
        anthrome_definitions = {
        0:  'Water',
        11: 'Urban',
        12: 'Mixed settlements',
        21: 'Rice villages',
        22: 'Irrigated villages',
        23: 'Rainfed villages',
        24: 'Pastoral villages',
        31: 'Residential irrigated croplands',
        32: 'Residential rainfed croplands',
        33: 'Populated croplands',
        34: 'Remote croplands',
        41: 'Residential rangelands',
        42: 'Populated rangelands',
        43: 'Remote rangelands',
        51: 'Residential woodlands',
        52: 'Populated woodlands',
        53: 'Remote woodlands',
        54: 'Inhabited treeless and barren lands',
        61: 'Wild woodlands',
        62: 'Wild treeless and barren lands'
        }

        #load UMBC Anthrome grid
        UMBC_Anthrome_root = Dataset(self.run_details['data_root']+'globally_gridded_metadata/ANTHROME/UMBC_ANTHROME.nc')
        UMBC_Anthrome_root.set_auto_mask(False)

        #get lon/lat centres/edges
        grid_edge_lons = UMBC_Anthrome_root['lon_e'][:]
        grid_edge_lats = UMBC_Anthrome_root['lat_e'][:]
        grid_centre_lons = UMBC_Anthrome_root['lon'][:]
        grid_centre_lats = UMBC_Anthrome_root['lat'][:]

        #read data variable frame
        UMBC_Anthrome_classes = UMBC_Anthrome_root.variables['CLASS']

        #set relevant metadata fields for function
        relevant_metadata_fields = ['UMBC_anthrome_classification', 'UMBC_modal_anthrome_classification_5km', 'UMBC_modal_anthrome_classification_25km']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #define radiuses of metadata classifications that will be calculated
        radius_var_dict = {'UMBC_modal_anthrome_classification_5km':5.0, 'UMBC_modal_anthrome_classification_25km':25.0}

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get gridcell indices that longitude/latitude coordinates are contained in  
            lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

            #get gridcell specific 'UMBC_anthrome_classification' code 
            gridcell_UMBC_Anthrome_class_code = UMBC_Anthrome_classes[lat_i,lon_i].item()

            #a correction for coastal sites is made:
            # -- if class is water (i.e. code 0), then the modal code of the surrounding gridcells is used instead (if there is a tie, the lowest code is kept preferentially)
            # -- if the site is truly an ocean site, all the surrounding gridcells will be water also, and therefore the class will be maintained as water
            if gridcell_UMBC_Anthrome_class_code == 0:

                #get neighbouring gridcell anthrome class codes (all grid cells which share a boundary with the gridcell, including diagonally)
                neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices =  get_neighbouring_gridcell_indices(lon_i,lat_i,len(grid_centre_lons),len(grid_centre_lats))
                neighbouring_UMBC_Anthrome_class_codes = []
                for neighbour_lon_i, neighbour_lat_i in zip(neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices):
                    neighbouring_UMBC_Anthrome_class_codes.append(UMBC_Anthrome_classes[neighbour_lat_i,neighbour_lon_i].item())  

                #get minumum neighbouring_UMBC Anthrome class code - use this to overwrite original gridcell_UMBC_Anthrome_class_code
                gridcell_UMBC_Anthrome_class_code, _ = stats.mode(neighbouring_UMBC_Anthrome_class_codes)

            #set gridcell classification as string associated with code 
            self.metadata['UMBC_anthrome_classification'][relevant_metadata_indices] = anthrome_definitions[gridcell_UMBC_Anthrome_class_code]

            #egt the modal classifications from the gridcells fractions which intersect in a radius of 5km/25km around the station location
            for vv, radius_var in enumerate(list(radius_var_dict.keys())):

                #get relevant indices of model which intersect with perimeter points
                relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_{}'.format(radius_var_dict[radius_var])],unique_pair['extreme_{}'.format(radius_var_dict[radius_var])],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
                relevant_grid_class_codes = []
                relevant_grid_weights = []
                for box_indices in relevant_grid_indices:
                    relevant_grid_class_codes.append(UMBC_Anthrome_classes[box_indices[1],box_indices[0]].item())
                    relevant_grid_weights.append(box_indices[2])

                #get weighted mode of classifications from all gridcells intersecting with perimeter, using the fraction of gridcells inside perimeter as weights 
                #if there is a tie, the lowest code is kept preferentially (i.e. most urban)
                modal_value = weighted_mode(relevant_grid_class_codes, relevant_grid_weights)

                #set 'UMBC_modal_anthrome_classification_5km' and 'UMBC_modal_anthrome_classification_25km'
                self.metadata[radius_var][relevant_metadata_indices] = anthrome_definitions[modal_value]

        #close netCDF
        UMBC_Anthrome_root.close()
        
        
    def get_ASTER(self):

        '''parse globally gridded ASTER v3 altitudes'''

        self.run_details['log_file_str'] += 'GETTING ASTER v3 METADATA\n'

        #set relevant metadata fields for function
        relevant_metadata_fields = ['ASTER_v3_altitude', 'altitude', 'measurement_altitude']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #define current altitude file in memory
        current_altitude_file = ''

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < -83.0) or (current_latitude > 83.0):
                continue

            #get netCDF tile filename that the current station longitude/latitude is contained within
            lon_fname_int = int(np.floor(current_longitude))
            if lon_fname_int < 0:
                lon_fname_dir = 'W'
                lon_fname_int = np.abs(lon_fname_int)
            else:
                lon_fname_dir = 'E'
            lon_fname_int = str(lon_fname_int)
            if len(lon_fname_int) == 1:
                lon_fname_int = '00{}'.format(lon_fname_int)
            elif len(lon_fname_int) == 2:
                lon_fname_int = '0{}'.format(lon_fname_int)

            lat_fname_int = int(np.floor(current_latitude))
            if lat_fname_int < 0:
                lat_fname_dir = 'S'
                lat_fname_int = np.abs(lat_fname_int)
            else:
                lat_fname_dir = 'N'
            lat_fname_int = str(lat_fname_int)
            if len(lat_fname_int) == 1:
                lat_fname_int = '0{}'.format(lat_fname_int)
            relevant_altitude_file = self.run_details['data_root']+'globally_gridded_metadata/ASTER_v3/ASTGTMV003_{}{}{}{}_dem.nc'.format(lat_fname_dir,lat_fname_int,lon_fname_dir,lon_fname_int)
        
            #test if relevant_altitude_file exists, if not it is because you tile is all ocean. Continue.
            if not os.path.isfile(relevant_altitude_file):
                continue

            #if relevant altitude file is not the same as the one in memory, then read new file 
            if relevant_altitude_file != current_altitude_file:

                #load ASTER v3 altitude grid
                ASTER_root = Dataset(relevant_altitude_file)
                ASTER_root.set_auto_mask(False)    

                #get lon/lat centres/edges
                grid_centre_lons = ASTER_root['lon'][:]
                grid_centre_lats = ASTER_root['lat'][:]
                lon_res = np.abs(np.average(np.diff(grid_centre_lons)))
                lat_res = np.abs(np.average(np.diff(grid_centre_lats)))
                grid_edge_lons = grid_centre_lons - lon_res
                grid_edge_lons = np.append(grid_edge_lons, np.array(grid_centre_lons[-1]+lon_res, dtype=np.float32))
                grid_edge_lats = grid_centre_lats - lat_res
                grid_edge_lats = np.append(grid_edge_lats, np.array(grid_centre_lats[-1]+lat_res, dtype=np.float32))

                #read data variable frame
                ASTER_altitudes_var = ASTER_root['ASTER_GDEM_DEM']

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get gridcell indices that longitude/latitude coordinates are contained in  
            lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

            #get gridcell specific 'ASTER_v3_altitude' 
            gridcell_ASTER_altitude = ASTER_altitudes_var[lat_i,lon_i].item()
        
            #set 'ASTER_v3_altitude' (if not equal to -9999 == void pixel value)
            if gridcell_ASTER_altitude != -9999:
                self.metadata['ASTER_v3_altitude'][relevant_metadata_indices] = gridcell_ASTER_altitude

                #if current altitudes == np.NaN, this is because there was no valid altitude and measurement altitude metadata through time record. Overwrite with current ASTER v3 altitude estimation.
                #also set measurement altitudes. If sampling height is available this is = ASTER v3 altitude + sampling height; or if not solely ASTER v3 altitude 
                #write appropriate warnings to inform of changes
                for meta_ii in relevant_metadata_indices:

                    current_altitude =             self.metadata['altitude'][meta_ii]
                    current_sampling_height =      self.metadata['sampling_height'][meta_ii]
                    current_measurement_altitude = self.metadata['measurement_altitude'][meta_ii]

                    if pd.isnull(current_altitude):
                        self.metadata['altitude'][meta_ii] = gridcell_ASTER_altitude
                        #write warning to inform have estimated altitude with ASTER v3 altitude
                        self.metadata['process_warnings'][meta_ii] += 'No Altitude Metadata - Used ASTER v3 DEM to Estimate Altitude. '

                        #have sampling height?
                        if not pd.isnull(current_sampling_height):
                            self.metadata['measurement_altitude'][meta_ii] = gridcell_ASTER_altitude + current_sampling_height
                            self.metadata['process_warnings'][meta_ii] += 'No Measurement Altitude Metadata - Used ASTER v3 DEM + Sampling Height to Estimate Altitude. '
                        #if not, set measurement altitude as ASTER v3 altitue
                        #also set sampling height as 0
                        else:
                            self.metadata['measurement_altitude'][meta_ii] = gridcell_ASTER_altitude
                            self.metadata['sampling_height'][meta_ii] = 0.0
                            self.metadata['process_warnings'][meta_ii] += 'No Measurement Altitude Metadata - Used ASTER v3 DEM to Estimate Altitude. '

            #set current altitude file to be relevant altitude file
            current_altitude_file = copy.deepcopy(relevant_altitude_file)
    
        
    def get_ETOPO1(self):

        '''parse globally gridded ETOPO1 altitudes'''

        self.run_details['log_file_str'] += 'GETTING ETOPO1 METADATA\n'

        #load ETOPO1 altitude grid
        ETOPO1_root = Dataset(self.run_details['data_root']+'globally_gridded_metadata/ETOPO1/ETOPO1.nc')
        ETOPO1_root.set_auto_mask(False)    

        #get lon/lat centres/edges
        grid_edge_lons = ETOPO1_root['lon_e'][:]
        grid_edge_lats = ETOPO1_root['lat_e'][:]
        grid_centre_lons = ETOPO1_root['lon'][:]
        grid_centre_lats = ETOPO1_root['lat'][:]

        #read data variable frame
        ETOPO1_altitudes_var = ETOPO1_root['Z']

        #set relevant metadata fields for function
        relevant_metadata_fields = ['ETOPO1_altitude', 'ETOPO1_max_altitude_difference_5km', 'altitude', 'measurement_altitude']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get gridcell indices that longitude/latitude coordinates are contained in  
            lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

            #get gridcell specific 'ETOPO1_altitude' 
            gridcell_ETOPO1_altitude = ETOPO1_altitudes_var[lat_i,lon_i].item()

            #a correction for coastal sites is made:
            # -- if the derived altitude is <= -5 m, the maximum altitude of the neighbouring grid cells will be used instead. 
            # -- if all neighbouring grid cells have altitudes <= -5 m, the original value will be retained.
            if gridcell_ETOPO1_altitude <= -5.0:
        
                #get neighbouring gridcell altitudes (all grid cells which share a boundary with the gridcell, including diagonally)
                neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices =  get_neighbouring_gridcell_indices(lon_i,lat_i,len(grid_centre_lons),len(grid_centre_lats))
                neighbouring_ETOPO1_altitudes = []
                for neighbour_lon_i, neighbour_lat_i in zip(neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices):
                    neighbouring_ETOPO1_altitudes.append(ETOPO1_altitudes_var[neighbour_lat_i,neighbour_lon_i].item())                 

                #take maximum neighbouring ETOPO1 altitude
                max_neighbouring_ETOPO1_altitude = np.max(neighbouring_ETOPO1_altitudes)                 

                #if all neighbouring grid cells have altitudes <= -5 m, maintain original value
                if max_neighbouring_ETOPO1_altitude > -5.0:
                    gridcell_ETOPO1_altitude = copy.deepcopy(max_neighbouring_ETOPO1_altitude)
        
            #set 'ETOPO1_altitude'
            self.metadata['ETOPO1_altitude'][relevant_metadata_indices] = gridcell_ETOPO1_altitude

            #calculate 'ETOPO1_max_altitude_difference_5km'
            #This is the altitude difference between the previously calcluated ETOPO1_altitude, and the minimum ETOP1 altitude (in metres) from all gridcells where more or equal to 50% of the cell intersects with a geodesic perimeter of 5 km around the station location.      

            #get relevant indices of model which intersect with perimeter points
            relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_5.0'],unique_pair['extreme_5.0'],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
            relevant_grid_altitudes = []
            for box_indices in relevant_grid_indices:
                #if gridcell intersection with perimeter is >= than 50%, then gridcell is relevant
                if box_indices[2] >= 0.5:
                    relevant_grid_altitudes.append(ETOPO1_altitudes_var[box_indices[1],box_indices[0]].item())

            #calculate difference between minumum altitude in relevant gridcells and ETOPO1_altitude
            gridcell_ETOPO1_max_altitude_difference_5km = gridcell_ETOPO1_altitude - np.min(relevant_grid_altitudes)

            #set 'ETOPO1_max_altitude_difference_5km'
            self.metadata['ETOPO1_max_altitude_difference_5km'][relevant_metadata_indices] = gridcell_ETOPO1_max_altitude_difference_5km

            #if current altitudes == np.NaN, this is because there was no valid altitude and measurement altitude metadata through time record, and no ASTER v3 DEM altitude information available. Overwrite with current ETOPO1 altitude estimation.
            #also set measurement altitudes. If sampling height is available this is = ETOPO1 altitude + sampling height; or if not solely ETOP01 altitude 
            #write appropriate warnings to inform of changes
            for meta_ii in relevant_metadata_indices:

                current_altitude =             self.metadata['altitude'][meta_ii]
                current_sampling_height =      self.metadata['sampling_height'][meta_ii]
                current_measurement_altitude = self.metadata['measurement_altitude'][meta_ii]

                if pd.isnull(current_altitude):
                    self.metadata['altitude'][meta_ii] = gridcell_ETOPO1_altitude
                    #write warning to inform have estimated altitude with ETOPO1 altitude
                    self.metadata['process_warnings'][meta_ii] += 'No Altitude Metadata - Used ETOPO1 to Estimate Altitude. '

                    #have sampling height?
                    if not pd.isnull(current_sampling_height):
                        self.metadata['measurement_altitude'][meta_ii] = gridcell_ETOPO1_altitude + current_sampling_height
                        self.metadata['process_warnings'][meta_ii] += 'No Measurement Altitude Metadata - Used ETOPO1 + Sampling Height to Estimate Altitude. '
                    #if not, set measurement altitude as ETOPO1 altitue
                    #also set sampling height as 0
                    else:
                        self.metadata['measurement_altitude'][meta_ii] = gridcell_ETOPO1_altitude
                        self.metadata['sampling_height'][meta_ii] = 0
                        self.metadata['process_warnings'][meta_ii] += 'No Measurement Altitude Metadata - Used ETOPO1 to Estimate Altitude. '

        #close netCDF
        ETOPO1_root.close()


    def get_GSFC_coastline_proximity(self):
        
        '''parse globally gridded GSFC coastline proximities'''

        self.run_details['log_file_str'] += 'GETTING GSFC COASTLINE PROXIMITY METADATA\n'

        #load GSFC coastline proximity grid (indexed by: latitude, longitude)
        GSFC_coastline_proximities = Dataset(self.run_details['data_root']+'globally_gridded_metadata/GMT_COAST_DISTANCE/GSFC_coastline_proximity.nc')
        GSFC_coastline_proximities.set_auto_mask(False)

        #get lon/lat centres/edges
        grid_edge_lons = GSFC_coastline_proximities['lon_e'][:]
        grid_edge_lats = GSFC_coastline_proximities['lat_e'][:]
        grid_centre_lons = GSFC_coastline_proximities['lon'][:]
        grid_centre_lats = GSFC_coastline_proximities['lat'][:]

        #read data variable frame
        coastline_proximities = GSFC_coastline_proximities['PROX']

        #set relevant metadata fields for function
        relevant_metadata_fields = ['GSFC_coastline_proximity']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get gridcell indices that longitude/latitude coordinates are contained in  
            lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

            #set gridcell specific 'GSFC_coastline_proximity' 
            self.metadata['GSFC_coastline_proximity'][relevant_metadata_indices] = coastline_proximities[lat_i,lon_i].item()


    def get_Koppen_Geiger_classification(self):

        '''parse Koppen-Geiger Classification'''

        self.run_details['log_file_str'] += 'GETTING KOPPEN-GEIGER CLASSIFICATION METADATA\n'

        #define all distinct Koppen-Geiger classifications by code:string
        koppen_geiger_definitions = {
        0: 'Water',
        1: 'Tropical - rainforest',
        2: 'Tropical - monsoon',
        3: 'Tropical - savannah',
        4: 'Arid - desert - hot',
        5: 'Arid - desert - cold',
        6: 'Arid - steppe - hot',
        7: 'Arid - steppe - cold',
        8: 'Temperate - dry summer - hot summer',
        9: 'Temperate - dry summer - warm summer',
        10: 'Temperate - dry summer - cold summer',
        11: 'Temperate - dry winter - hot summer',
        12: 'Temperate - dry winter - warm summer',
        13: 'Temperate - dry winter - cold summer',
        14: 'Temperate - no dry season - hot summer',
        15: 'Temperate - no dry season - warm summer',
        16: 'Temperate - no dry season - cold summer',
        17: 'Cold - dry summer - hot summer',
        18: 'Cold - dry summer - warm summer',
        19: 'Cold - dry summer - cold summer',
        20: 'Cold - dry summer - very cold winter',
        21: 'Cold - dry winter - hot summer',
        22: 'Cold - dry winter - warm summer',
        23: 'Cold - dry winter - cold summer',
        24: 'Cold - dry winter - very cold winter',
        25: 'Cold - no dry season - hot summer',
        26: 'Cold - no dry season - warm summer',
        27: 'Cold - no dry season - cold summer',
        28: 'Cold - no dry season - very cold winter',
        29: 'Polar - tundra',
        30: 'Polar - frost',
        }

        #load Koppen-geiger classification grid (indexed by: latitude, longitude)
        koppen_geiger_classes = Dataset(self.run_details['data_root']+'globally_gridded_metadata/KOPPEN_GEIGER/Koppen_Geiger.nc')
        koppen_geiger_classes.set_auto_mask(False)

        #get lon/lat centres/edges
        grid_edge_lons = koppen_geiger_classes['lon_e'][:]
        grid_edge_lats = koppen_geiger_classes['lat_e'][:]
        grid_centre_lons = koppen_geiger_classes['lon'][:]
        grid_centre_lats = koppen_geiger_classes['lat'][:]

        #read data variable frame
        classes = koppen_geiger_classes['CLASS']

        #set relevant metadata fields for function
        relevant_metadata_fields = ['Koppen-Geiger_classification', 'Koppen-Geiger_modal_classification_5km', 'Koppen-Geiger_modal_classification_25km']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #define radiuses of metadata classifications that will be calculated
        radius_var_dict = {'Koppen-Geiger_modal_classification_5km':5.0, 'Koppen-Geiger_modal_classification_25km':25.0}

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get gridcell indices that longitude/latitude coordinates are contained in  
            lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

            #get gridcell specific 'Koppen-Geiger' code 
            gridcell_Koppen_Geiger_class_code = int(classes[lat_i,lon_i].item())

            #a correction for coastal sites is made:
            # -- if class is water (i.e. code 0), then the modal code of the surrounding gridcells is used instead (if there is a tie, the lowest code is kept preferentially)
            # -- if the site is truly an ocean site, all the surrounding gridcells will be water also, and therefore the class will be maintained as water
            if gridcell_Koppen_Geiger_class_code == 0:

                #get neighbouring gridcell anthrome class codes (all grid cells which share a boundary with the gridcell, including diagonally)
                neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices =  get_neighbouring_gridcell_indices(lon_i,lat_i,len(grid_centre_lons),len(grid_centre_lats))
                neighbouring_Koppen_Geiger_class_codes = []
                for neighbour_lon_i, neighbour_lat_i in zip(neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices):
                    neighbouring_Koppen_Geiger_class_code = int(classes[neighbour_lat_i,neighbour_lon_i].item())
                    neighbouring_Koppen_Geiger_class_codes.append(neighbouring_Koppen_Geiger_class_code)  

                #get minumum neighbouring Koppen Geiger class code - use this to overwrite original gridcell_Koppen_Geiger_class_code
                gridcell_Koppen_Geiger_class_code, _ = stats.mode(neighbouring_Koppen_Geiger_class_codes)

            #set gridcell classification as string associated with code 
            self.metadata['Koppen-Geiger_classification'][relevant_metadata_indices] = koppen_geiger_definitions[gridcell_Koppen_Geiger_class_code]

            #egt the modal classifications from the gridcells fractions which intersect in a radius of 5km/25km around the station location
            for vv, radius_var in enumerate(list(radius_var_dict.keys())):

                #get relevant indices of model which intersect with perimeter points
                relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_{}'.format(radius_var_dict[radius_var])],unique_pair['extreme_{}'.format(radius_var_dict[radius_var])],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
                relevant_grid_class_codes = []
                relevant_grid_weights = []
                for box_indices in relevant_grid_indices:
                    relevant_gridbox_class_code = int(classes[box_indices[1],box_indices[0]].item())
                    relevant_grid_class_codes.append(relevant_gridbox_class_code)
                    relevant_grid_weights.append(box_indices[2])

                #get weighted mode of classifications from all gridcells intersecting with perimeter, using the fraction of gridcells inside perimeter as weights 
                #if there is a tie, the lowest code is kept preferentially 
                modal_value = weighted_mode(relevant_grid_class_codes, relevant_grid_weights)

                #set 'Koppen-Geiger_modal_classification_5km' and 'Koppen-Geiger_modal_classification_25km'
                self.metadata[radius_var][relevant_metadata_indices] = koppen_geiger_definitions[modal_value]

    def get_Meybeck_classification(self):

        '''parse ESDAC Meybeck terrain classifications'''

        self.run_details['log_file_str'] += 'GETTING MEYBECK TERRAIN CLASSIFICATION METADATA\n'

        #define all distinct Meybeck classifications by code:string
        Meybeck_definitions = {
        1:  'Plains',
        2:  'Mid Altitude Plains',
        3:  'High Altitude Plains',
        4:  'Lowlands',
        5:  'Rugged Lowlands',
        6:  'Very Low Plateaus',
        7:  'Low Plateaus',
        8:  'Mid Altitude Plateaus',
        9:  'High Altitude Plateaus',
        10:  'Very High Altitude Plateaus',
        11:  'Hills',
        12:  'Low Altitude Mountains',
        13:  'Mid Altitude Mountains',
        14:  'High Altitude Mountains',
        15:  'Very High Altitude Mountains',
        255: 'Water'
        }

        #load Meybeck classification grid (indexed by: latitude, longitude)
        meybeck_classes = Dataset(self.run_details['data_root']+'globally_gridded_metadata/EU_LANDFORM_CLASSIFICATION/Meybeck.nc')
        meybeck_classes.set_auto_mask(False)

        #get lon/lat centres/edges
        grid_edge_lons = meybeck_classes['lon_e'][:]
        grid_edge_lats = meybeck_classes['lat_e'][:]
        grid_centre_lons = meybeck_classes['lon'][:]
        grid_centre_lats = meybeck_classes['lat'][:]

        #read data variable frame
        classes = meybeck_classes['CLASS']

        #set relevant metadata fields for function
        relevant_metadata_fields = ['ESDAC_Meybeck_landform_classification', 'ESDAC_modal_Meybeck_landform_classification_5km', 'ESDAC_modal_Meybeck_landform_classification_25km']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #define radiuses of metadata classifications that will be calculated
        radius_var_dict = {'ESDAC_modal_Meybeck_landform_classification_5km':5.0, 'ESDAC_modal_Meybeck_landform_classification_25km':25.0}

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get gridcell indices that longitude/latitude coordinates are contained in  
            lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

            #get gridcell specific 'Meybeck' code 
            gridcell_meybeck_class_code = int(classes[lat_i,lon_i].item())

            #a correction for coastal sites is made:
            # -- if class is water (i.e. code 255), then the modal code of the surrounding gridcells is used instead (if there is a tie, the lowest code is kept preferentially)
            # -- if the site is truly an ocean site, all the surrounding gridcells will be water also, and therefore the class will be maintained as water
            if gridcell_meybeck_class_code == 255:

                #get neighbouring gridcell anthrome class codes (all grid cells which share a boundary with the gridcell, including diagonally)
                neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices =  get_neighbouring_gridcell_indices(lon_i,lat_i,len(grid_centre_lons),len(grid_centre_lats))
                neighbouring_meybeck_class_codes = []
                for neighbour_lon_i, neighbour_lat_i in zip(neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices):
                    neighbouring_meybeck_class_code = int(classes[neighbour_lat_i,neighbour_lon_i].item())
                    neighbouring_meybeck_class_codes.append(neighbouring_meybeck_class_code)  

                #get minumum neighbouring Meybeck class code - use this to overwrite original gridcell_meybeck_class_code
                gridcell_meybeck_class_code, _ = stats.mode(neighbouring_meybeck_class_codes)

            #set gridcell classification as string associated with code 
            self.metadata['ESDAC_Meybeck_landform_classification'][relevant_metadata_indices] = Meybeck_definitions[gridcell_meybeck_class_code]

            #egt the modal classifications from the gridcells fractions which intersect in a radius of 5km/25km around the station location
            for vv, radius_var in enumerate(list(radius_var_dict.keys())):

                #get relevant indices of model which intersect with perimeter points
                relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_{}'.format(radius_var_dict[radius_var])],unique_pair['extreme_{}'.format(radius_var_dict[radius_var])],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
                relevant_grid_class_codes = []
                relevant_grid_weights = []
                for box_indices in relevant_grid_indices:
                    relevant_gridbox_class_code = int(classes[box_indices[1],box_indices[0]].item())
                    relevant_grid_class_codes.append(relevant_gridbox_class_code)
                    relevant_grid_weights.append(box_indices[2])

                #get weighted mode of classifications from all gridcells intersecting with perimeter, using the fraction of gridcells inside perimeter as weights 
                #if there is a tie, the lowest code is kept preferentially 
                modal_value = weighted_mode(relevant_grid_class_codes, relevant_grid_weights)

                #set 'ESDAC_modal_Meybeck_landform_classification_5km' and 'ESDAC_modal_Meybeck_landform_classification_25km'
                self.metadata[radius_var][relevant_metadata_indices] = Meybeck_definitions[modal_value]


    def get_Iwahashi_classification(self):

        '''parse ESDAC Iwahashi terrain classifications'''

        self.run_details['log_file_str'] += 'GETTING IWAHASHI TERRAIN CLASSIFICATION METADATA\n'

        #define all distinct Iwahashi classifications by code:string
        Iwahashi_definitions = {
        1:  'steep - fine texture - high convexity',
        2:  'steep - coarse texture - high convexity',
        3:  'steep - fine texture - low convexity',
        4:  'steep - coarse texture - low convexity',
        5:  'medium steep - fine texture - high convexity',
        6:  'medium steep - coarse texture - high convexity',
        7:  'medium steep - fine texture - low convexity',
        8:  'medium steep - coarse texture - low convexity',
        9:  'medium gentle - fine texture - high convexity',
        10:  'medium gentle - coarse texture - high convexity',
        11:  'medium gentle - fine texture - low convexity',
        12:  'medium gentle - coarse texture - low convexity',
        13:  'gentle - fine texture - high convexity',
        14:  'gentle - coarse texture - high convexity',
        15:  'gentle - fine texture - low convexity',
        16:  'gentle - coarse texture - low convexity',
        255: 'Water'
        }

        #load Iwahashi classification grid (indexed by: latitude, longitude)
        iwahashi_classes = Dataset(self.run_details['data_root']+'globally_gridded_metadata/EU_LANDFORM_CLASSIFICATION/Iwahashi.nc')
        iwahashi_classes.set_auto_mask(False)

        #get lon/lat centres/edges
        grid_edge_lons = iwahashi_classes['lon_e'][:]
        grid_edge_lats = iwahashi_classes['lat_e'][:]
        grid_centre_lons = iwahashi_classes['lon'][:]
        grid_centre_lats = iwahashi_classes['lat'][:]

        #read data variable frame
        classes = iwahashi_classes['CLASS']   

        #set relevant metadata fields for function
        relevant_metadata_fields = ['ESDAC_Iwahashi_landform_classification', 'ESDAC_modal_Iwahashi_landform_classification_5km', 'ESDAC_modal_Iwahashi_landform_classification_25km']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #define radiuses of metadata classifications that will be calculated
        radius_var_dict = {'ESDAC_modal_Iwahashi_landform_classification_5km':5.0, 'ESDAC_modal_Iwahashi_landform_classification_25km':25.0}

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #if current longitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_longitude < grid_edge_lons[0]) or (current_longitude > grid_edge_lons[-1]):
                continue

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue
     
            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get gridcell indices that longitude/latitude coordinates are contained in  
            lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

            #get gridcell specific 'Iwahashi' code 
            gridcell_iwahashi_class_code = int(classes[lat_i,lon_i].item())

            #a correction for coastal sites is made:
            # -- if class is water (i.e. code 255), then the modal code of the surrounding gridcells is used instead (if there is a tie, the lowest code is kept preferentially)
            # -- if the site is truly an ocean site, all the surrounding gridcells will be water also, and therefore the class will be maintained as water
            if gridcell_iwahashi_class_code == 255:

                #get neighbouring gridcell anthrome class codes (all grid cells which share a boundary with the gridcell, including diagonally)
                neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices =  get_neighbouring_gridcell_indices(lon_i,lat_i,len(grid_centre_lons),len(grid_centre_lats))
                neighbouring_iwahashi_class_codes = []
                for neighbour_lon_i, neighbour_lat_i in zip(neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices):
                    neighbouring_iwahashi_class_code = int(classes[neighbour_lat_i,neighbour_lon_i].item())
                    neighbouring_iwahashi_class_codes.append(neighbouring_iwahashi_class_code)  

                #get minumum neighbouring Iwahashi class code - use this to overwrite original gridcell_iwahashi_class_code
                gridcell_iwahashi_class_code, _ = stats.mode(neighbouring_iwahashi_class_codes)

            #set gridcell classification as string associated with code 
            self.metadata['ESDAC_Iwahashi_landform_classification'][relevant_metadata_indices] = Iwahashi_definitions[gridcell_iwahashi_class_code]

            #egt the modal classifications from the gridcells fractions which intersect in a radius of 5km/25km around the station location
            for vv, radius_var in enumerate(list(radius_var_dict.keys())):

                #get relevant indices of model which intersect with perimeter points
                relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_{}'.format(radius_var_dict[radius_var])],unique_pair['extreme_{}'.format(radius_var_dict[radius_var])],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
                relevant_grid_class_codes = []
                relevant_grid_weights = []
                for box_indices in relevant_grid_indices:
                    relevant_gridbox_class_code = int(classes[box_indices[1],box_indices[0]].item())
                    relevant_grid_class_codes.append(relevant_gridbox_class_code)
                    relevant_grid_weights.append(box_indices[2])

                #get weighted mode of classifications from all gridcells intersecting with perimeter, using the fraction of gridcells inside perimeter as weights 
                #if there is a tie, the lowest code is kept preferentially 
                modal_value = weighted_mode(relevant_grid_class_codes, relevant_grid_weights)

                #set 'ESDAC_modal_Iwahashi_landform_classification_5km' and 'ESDAC_modal_Iwahashi_landform_classification_25km'
                self.metadata[radius_var][relevant_metadata_indices] = Iwahashi_definitions[modal_value]


    def get_MODIS_classifications(self): 

        '''get MODIS land use (IGBP and UMD) and Leaf Area Index classifications'''

        self.run_details['log_file_str'] += 'GETTING MODIS LAND USE/LAI METADATA\n'

        #define all MODIS dataset classifications
        class_definitions = {
            #define all distinct MODIS IGBP land use classifications by code to name/description
            'IGBP_land_use' : {
            0: {'name':'Water Bodies', 'description':'At least 60% of area is covered by permanent water bodies.'}, 
            1: {'name':'Evergreen Needleleaf Forests', 'description':'Dominated by evergreen conifer trees (canopy >2m). Tree cover >60%.'},
            2: {'name':'Evergreen Broadleaf Forests', 'description':'Dominated by evergreen broadleaf and palmate trees (canopy >2m). Tree cover >60%.'},
            3: {'name':'Deciduous Needleleaf Forests', 'description':'Dominated by deciduous needleleaf (larch) trees (canopy >2m). Tree cover >60%.'},
            4: {'name':'Deciduous Broadleaf Forests', 'description':'Dominated by deciduous broadleaf trees (canopy >2m). Tree cover >60%.'},
            5: {'name':'Mixed Forests', 'description':'Dominated by neither deciduous nor evergreen (40-60% of each) tree type (canopy >2m). Tree cover >60%'},
            6: {'name':'Closed Shrublands', 'description':'Dominated by woody perennials (1-2m height) >60% cover.'},
            7: {'name':'Open Shrublands', 'description':'Dominated by woody perennials (1-2m height) 10-60% cover.'},
            8: {'name':'Woody Savannas', 'description':'Tree cover 30-60% (canopy >2m)'},   
            9: {'name':'Savannas', 'description':'Tree cover 10-30% (canopy >2m).'},
            10: {'name':'Grasslands', 'description':'Dominated by herbaceous annuals (<2m).'},
            11: {'name':'Permanent Wetlands', 'description':'Permanently inundated lands with 30-60% water cover and >10% vegetated cover.'},
            12: {'name':'Croplands', 'description':'At least 60% of area is cultivated cropland.'},
            13: {'name':'Urban and Built-up Lands', 'description':'At least 30% impervious surface area including building materials, asphalt, and vehicles.'},
            14: {'name':'Cropland/Natural Vegetation Mosaics', 'description':'Mosaics of small-scale cultivation 40-60% with natural tree, shrub, or herbaceous vegetation.'},
            15: {'name':'Permanent Snow and Ice', 'description':'At least 60% of area is covered by snow and ice for at least 10 months of the year.'},
            16: {'name':'Barren', 'description':'At least 60% of area is non-vegetated barren (sand, rock, soil) areas with less than 10% vegetation.'},
            255: {'name':'Unclassified', 'description':'Has not received a map label because of missing inputs.'}
            },    

            #define all distinct MODIS University of Maryland (UMD) land use classifications by code to name/description
            'UMD_land_use' : {
            0: {'name':'Water Bodies', 'description':'At least 60% of area is covered by permanent water bodies.'},
            1: {'name':'Evergreen Needleleaf Forests', 'description':'Dominated by evergreen conifer trees (canopy >2m). Tree cover >60%.'},
            2: {'name':'Evergreen Broadleaf Forests', 'description':'Dominated by evergreen broadleaf and palmate trees (canopy >2m). Tree cover >60%.'},
            3: {'name':'Deciduous Needleleaf Forests', 'description':'Dominated by deciduous needleleaf (larch) trees (canopy >2m). Tree cover >60%.'},
            4: {'name':'Deciduous Broadleaf Forests', 'description':'Dominated by deciduous broadleaf trees (canopy >2m). Tree cover >60%.'},
            5: {'name':'Mixed Forests', 'description':'Dominated by neither deciduous nor evergreen (40-60% of each) tree type (canopy >2m). Tree cover >60%'},
            6: {'name':'Closed Shrublands', 'description':'Dominated by woody perennials (1-2m height) >60% cover.'},
            7: {'name':'Open Shrublands', 'description':'Dominated by woody perennials (1-2m height) 10-60% cover.'},
            8: {'name':'Woody Savannas', 'description':'Tree cover 30-60% (canopy >2m)'},
            9: {'name':'Savannas', 'description':'Tree cover 10-30% (canopy >2m).'},
            10: {'name':'Grasslands', 'description':'Dominated by herbaceous annuals (<2m).'},
            11: {'name':'Permanent Wetlands', 'description':'Permanently inundated lands with 30-60% water cover and >10% vegetated cover.'},
            12: {'name':'Croplands', 'description':'At least 60% of area is cultivated cropland.'},
            13: {'name':'Urban and Built-up Lands', 'description':'At least 30% impervious surface area including building materials, asphalt, and vehicles.'},
            14: {'name':'Cropland/Natural Vegetation Mosaics', 'description':'Mosaics of small-scale cultivation 40-60% with natural tree, shrub, or herbaceous vegetation.'},
            15: {'name':'Non-Vegetated Lands', 'description':'At least 60% of area is non-vegetated barren (sand, rock, soil) or permanent snow and ice with less than 10% vegetation.'},
            255: {'name':'Unclassified', 'description':'Has not received a map label because of missing inputs.'}
            },

            #define all distinct MODIS Leaf Area Index (LAII) classifications by code to name/description
            'LAI' : {
            0: {'name':'Water Bodies', 'description':'At least 60% of area is covered by permanent water bodies.'},
            1: {'name':'Grasslands', 'description':'Dominated by herbaceous annuals (<2m) including cereal croplands.'},
            2: {'name':'Shrublands', 'description':'Shrub (1-2m) cover >10%.'},
            3: {'name':'Broadleaf Croplands', 'description':'Dominated by herbaceous annuals (<2m) that are cultivated with broadleaf crops.'},
            4: {'name':'Savannas', 'description':'Between 10-60% tree cover (>2m).'},
            5: {'name':'Evergreen Broadleaf Forests', 'description':'Dominated by evergreen broadleaf and palmate trees (>2m). Tree cover >60%'},
            6: {'name':'Deciduous Broadleaf Forests', 'description':'Dominated by deciduous broadleaf trees (>2m). Tree cover >60%.'},
            7: {'name':'Evergreen Needleleaf Forests', 'description':'Dominated by evergreen conifer trees (>2m). Tree cover >60%.'},
            8: {'name':'Deciduous Needleleaf Forests', 'description':'Dominated by deciduous needleleaf (larch) trees (>2m). Tree cover >60%.'},
            9: {'name':'Non-Vegetated Lands', 'description':'At least 60% of area is non-vegetated barren (sand, rock, soil) or permanent snow and ice with less than 10% vegetation.'},
            10: {'name':'Urban and Built-up Lands', 'description':'At least 30% impervious surface area including building materials, asphalt, and vehicles.'},
            255: {'name':'Unclassified', 'description':'Has not received a map label because of missing inputs.'}
            }
        }

        #iterate through all MODIS classification datasets, processing each one by one
        for dataset in ['IGBP_land_use','UMD_land_use','LAI']:

            #get all year files for specific classification dataset
            class_files = sorted(glob.glob(self.run_details['data_root']+'globally_gridded_metadata/MODIS_MCD12C1/MODIS_MCD12C1_v6_{}*'.format(dataset)))
            #get years of all classification dataset files 
            class_years = [int(class_file.split(dataset+'_')[-1].split('.nc')[0]) for class_file in class_files]
            #convert years to datetimes (in midpoint of year --> July 2nd 12:00 for non-leap years, July 2nd 00:00 for leap years)
            class_datetimes = [datetime.datetime(year, 7, 2, 0, 0, 0) if calendar.isleap(year) else datetime.datetime(year, 7, 2, 12, 0, 0) for year in class_years]        
            #convert datetimes to minutes since reference
            class_times = np.array([datetime_to_fractional_minutes_from_reference(class_datetime) for class_datetime in class_datetimes])
                
            #set relevant metadata fields for function
            relevant_metadata_fields = ['MODIS_MCD12C1_v6_{}'.format(dataset), 'MODIS_MCD12C1_v6_modal_{}_5km'.format(dataset), 'MODIS_MCD12C1_v6_modal_{}_25km'.format(dataset)]
            self.metadata_fields_to_update.extend(relevant_metadata_fields)

            #define radiuses of metadata classifications that will be calculated
            radius_var_dict = {'MODIS_MCD12C1_v6_modal_{}_5km'.format(dataset):5.0, 'MODIS_MCD12C1_v6_modal_{}_25km'.format(dataset):25.0}
        
            #define current class file in memory
            current_class_file = ''

            #initialise previous longitude/latitude
            previous_longitude = -999.9
            previous_latitude = -999.9

            #iterate through all metadata columns
            for metadata_index, metadata_column in enumerate(self.metadata):

                #get current longitude/latitude
                current_longitude = metadata_column['longitude']
                current_latitude = metadata_column['latitude']
                #get unique pair dictionary for longitude/latitude pair
                for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():
                    if (unique_pair['longitude'] == current_longitude) & (unique_pair['latitude'] == current_latitude):
                        break
    
                #get the midpoint of the metadata window
                if metadata_index != (len(self.metadata['latitude']) - 1):
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + ((self.metadata['meta_update_stamp'][metadata_index+1]-self.metadata['meta_update_stamp'][metadata_index])/2.)
                else:
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + (((self.data['time'][-1] + self.data['temporal_resolution'][-1])-self.metadata['meta_update_stamp'][metadata_index])/2.)    

                #find which data file has data closest to the metadata midpoint
                relevant_class_file = class_files[np.abs(class_times - metadata_midpoint).argmin()]
                
                #if relevant class file is not the same as the one in memory, then read new file 
                if relevant_class_file != current_class_file:
    
                    #read class file frame (indexed by: latitude, longitude)
                    loaded_classes = Dataset(relevant_class_file)
                    loaded_classes.set_auto_mask(False)
                
                    #get lon/lat centres/edges
                    grid_edge_lons = loaded_classes['lon_e'][:]
                    grid_edge_lats = loaded_classes['lat_e'][:]
                    grid_centre_lons = loaded_classes['lon'][:]
                    grid_centre_lats = loaded_classes['lat'][:]

                    #read data variable frame
                    classes = loaded_classes['CLASS']     

                #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
                if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                    continue

                #if class file is not the same, or longitude or latitude have changed then 
                #get gridcell indices that longitude/latitude coordinates are contained in  
                if (relevant_class_file != current_class_file) or (previous_longitude != current_longitude) or (previous_latitude != current_latitude):
                    lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

                    #get gridcell specific code
                    gridcell_class_code = int(classes[lat_i,lon_i].item())

                    #a correction for coastal sites is made:
                    # -- if class is water (i.e. code 0), then the modal code of the surrounding gridcells is used instead (if there is a tie, the lowest code is kept preferentially)
                    # -- if the site is truly an ocean site, all the surrounding gridcells will be water also, and therefore the class will be maintained as water
                    if gridcell_class_code == 0:

                        #get neighbouring gridcell anthrome class codes (all grid cells which share a boundary with the gridcell, including diagonally)
                        neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices =  get_neighbouring_gridcell_indices(lon_i,lat_i,len(grid_centre_lons),len(grid_centre_lats))
                        neighbouring_class_codes = []
                        for neighbour_lon_i, neighbour_lat_i in zip(neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices):
                            neighbouring_class_code = int(classes[neighbour_lat_i,neighbour_lon_i].item())
                            neighbouring_class_codes.append(neighbouring_class_code)  

                        #get minumum neighbouring class code - use this to overwrite original gridcell class_code
                        gridcell_class_code, _ = stats.mode(neighbouring_class_codes)

                    #get the modal classifications from the gridcells fractions which intersect in a radius of 5km/25km around the station location
                    #initialise dictionaries for storing model codes
                    modal_codes = {}
                    for vv, radius_var in enumerate(list(radius_var_dict.keys())):

                        #get relevant indices of model which intersect with perimeter points
                        relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_{}'.format(radius_var_dict[radius_var])],unique_pair['extreme_{}'.format(radius_var_dict[radius_var])],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
                        relevant_grid_class_codes = []
                        relevant_grid_weights = []
                        for box_indices in relevant_grid_indices:
                            relevant_gridbox_class_code = int(classes[box_indices[1],box_indices[0]].item())
                            relevant_grid_class_codes.append(relevant_gridbox_class_code)
                            relevant_grid_weights.append(box_indices[2])

                        #get weighted mode of classifications from all gridcells intersecting with perimeter, using the fraction of gridcells inside perimeter as weights 
                        #if there is a tie, the lowest code is kept preferentially 
                        modal_codes[radius_var] = weighted_mode(relevant_grid_class_codes, relevant_grid_weights)

                #set gridcell classifications (as strings associated with codes)
                self.metadata['MODIS_MCD12C1_v6_{}'.format(dataset)][metadata_index] = class_definitions[dataset][gridcell_class_code]['name']
                for vv, radius_var in enumerate(list(radius_var_dict.keys())):
                    #set 'modal_5km' and 'modal_25km' classifications
                    modal_code = modal_codes[radius_var]
                    self.metadata[radius_var][metadata_index] = class_definitions[dataset][modal_code]['name']

                #set current longitude/latitude as previous values
                previous_longitude = copy.deepcopy(current_longitude)
                previous_latitude = copy.deepcopy(current_latitude)

                #set current class file to be relevant class file
                current_class_file = copy.deepcopy(relevant_class_file)


    def get_GHSL(self): 

        '''get Global Human Settlement Layer (GHSL) area/population densities and settlement model classifications.
           Data is supplied in Mollweide Projection.
        '''

        self.run_details['log_file_str'] += 'GETTING GHSL AREA/POPULATION DENSITIES AND SETTLEMENT MODEL CLASSIFICATIONS\n'

        #define all GHSL settlement model classifications
        GHSL_definitions = {
        10:'Water',
        11:'Very Low Density Rural',
        12:'Low Density Rural',
        13:'Rural Cluster',
        21:'Suburban/Peri-Urban',
        22:'Semi-Dense Urban Cluster',
        23:'Dense Urban Cluster',
        30:'Urban Centre' 
        }

        #iterate through all GHSL datasets, processing each one by one
        for dataset in ['GHS_BUILT_LDSMT','GHS_POP_MT','GHS_SMOD_POPMT']:

            #get all year files for specific classification dataset
            class_files = sorted(glob.glob(self.run_details['data_root']+'globally_gridded_metadata/GHSL/GHSL_{}*'.format(dataset)))
            #get years of all classification dataset files 
            class_years = [int(class_file.split(dataset+'_')[-1].split('.nc')[0]) for class_file in class_files]
            #convert years to datetimes (in midpoint of year --> July 2nd 12:00 for non-leap years, July 2nd 00:00 for leap years)
            class_datetimes = [datetime.datetime(year, 7, 2, 0, 0, 0) if calendar.isleap(year) else datetime.datetime(year, 7, 2, 12, 0, 0) for year in class_years]        
            #convert datetimes to minutes since reference
            class_times = np.array([datetime_to_fractional_minutes_from_reference(class_datetime) for class_datetime in class_datetimes])
            
            #set relevant metadata fields for function and define radiuses of metadata classifications that will be calculated   
            if dataset == 'GHS_BUILT_LDSMT':
                relevant_metadata_fields = ['GHSL_built_up_area_density', 'GHSL_average_built_up_area_density_5km', 'GHSL_average_built_up_area_density_25km', 'GHSL_max_built_up_area_density_5km', 'GHSL_max_built_up_area_density_25km']
                radius_var_dict = {'GHSL_average_built_up_area_density_5km':5.0, 'GHSL_average_built_up_area_density_25km':25.0}
            elif dataset == 'GHS_POP_MT':
                relevant_metadata_fields = ['GHSL_population_density', 'GHSL_average_population_density_5km', 'GHSL_average_population_density_25km', 'GHSL_max_population_density_5km', 'GHSL_max_population_density_25km']
                radius_var_dict = {'GHSL_average_population_density_5km':5.0, 'GHSL_average_population_density_25km':25.0}
            elif dataset == 'GHS_SMOD_POPMT':
                relevant_metadata_fields = ['GHSL_settlement_model_classification', 'GHSL_modal_settlement_model_classification_5km', 'GHSL_modal_settlement_model_classification_25km']
                radius_var_dict = {'GHSL_modal_settlement_model_classification_5km':5.0, 'GHSL_modal_settlement_model_classification_25km':25.0}   
            self.metadata_fields_to_update.extend(relevant_metadata_fields)      
   
            #define current class file in memory
            current_class_file = ''

            #initialise previous longitude/latitude
            previous_longitude = -999.9
            previous_latitude = -999.9

            #define coordinate reference system of the projection (Mollweide) that the data grid uses
            mollweide_crs = ccrs.Mollweide(central_longitude=0,false_easting=0,false_northing=0)
            #define geographic coordinate CRS
            geographic_crs = ccrs.PlateCarree()

            #iterate through all metadata columns
            for metadata_index, metadata_column in enumerate(self.metadata):

                #get current longitude/latitude
                current_longitude = metadata_column['longitude']
                current_latitude = metadata_column['latitude']
                #get unique pair dictionary for longitude/latitude pair
                for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():
                    if (unique_pair['longitude'] == current_longitude) & (unique_pair['latitude'] == current_latitude):
                        break

                #convert geographic longitude/latitude coordinates to the CRS of the data grid projection (Mollweide)
                moll_longitude, moll_latitude = mollweide_crs.transform_point(current_longitude, current_latitude, geographic_crs)    

                #get the midpoint of the metadata window
                if metadata_index != (len(self.metadata['latitude']) - 1):
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + ((self.metadata['meta_update_stamp'][metadata_index+1]-self.metadata['meta_update_stamp'][metadata_index])/2.)
                else:
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + (((self.data['time'][-1] + self.data['temporal_resolution'][-1])-self.metadata['meta_update_stamp'][metadata_index])/2.)    

                #find which data file has data closest to the metadata midpoint
                relevant_class_file = class_files[np.abs(class_times - metadata_midpoint).argmin()]
                
                #if relevant class file is not the same as the one in memory, then read new file 
                if relevant_class_file != current_class_file:
    
                    #read class file frame (indexed by: latitude, longitude)
                    loaded_classes = Dataset(relevant_class_file)
                    loaded_classes.set_auto_mask(False)
                
                    #get x/y gridcell coordinate centres/edges 
                    grid_edge_lons = loaded_classes['x_e'][:]
                    grid_edge_lats = loaded_classes['y_e'][:]
                    grid_centre_lons = loaded_classes['x'][:]
                    grid_centre_lats = loaded_classes['y'][:]

                    #read data variable frame
                    classes = loaded_classes['Z']     

                #if current mollweide latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
                if (moll_latitude < grid_edge_lats[0]) or (moll_latitude > grid_edge_lats[-1]):
                    continue

                #if class file is not the same, or longitude or latitude have changed then 
                #get gridcell indices that longitude/latitude coordinates are contained in  
                if (relevant_class_file != current_class_file) or (previous_longitude != current_longitude) or (previous_latitude != current_latitude):
                    lon_i, lat_i = get_coordinate_gridcell_indices(moll_longitude,moll_latitude,grid_centre_lons,grid_centre_lats)

                    #dataset provides classification?
                    if dataset == 'GHS_SMOD_POPMT':
                        #get gridcell specific code
                        gridcell_class_code = int(classes[lat_i,lon_i].item())

                        #a correction for coastal sites is made:
                        # -- if class is water (i.e. code 10), then the modal code of the surrounding gridcells is used instead (if there is a tie, the lowest code is kept preferentially)
                        # -- if the site is truly an ocean site, all the surrounding gridcells will be water also, and therefore the class will be maintained as water
                        # -- it is possible that some gridboxes will be out of projection (identified by negative values), if so then get correct wrapped gridbox neighbour instead
                        if gridcell_class_code == 10:
                            #get neighbouring gridcell anthrome class codes (all grid cells which share a boundary with the gridcell, including diagonally)
                            neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices =  get_neighbouring_gridcell_indices(lon_i,lat_i,len(grid_centre_lons),len(grid_centre_lats), projection_crs=mollweide_crs, grid_centre_lons=grid_centre_lons, grid_centre_lats=grid_centre_lats)
                            neighbouring_class_codes = []
                            for neighbour_lon_i, neighbour_lat_i in zip(neighbouring_gridcell_lon_indices, neighbouring_gridcell_lat_indices):
                                neighbouring_class_code = int(classes[neighbour_lat_i,neighbour_lon_i].item())
                                neighbouring_class_codes.append(neighbouring_class_code) 

                            #get minumum neighbouring class code - use this to overwrite original gridcell class code
                            gridcell_class_code, _ = stats.mode(neighbouring_class_codes)
                
                    #dataset provides numeric value
                    else:
                        #get gridcell specific value
                        gridcell_value = classes[lat_i,lon_i].item()

                    #get the modal classifications/average/max values from the gridcells fractions which intersect in a radius of 5km/25km around the station location
                    modal_codes = {}
                    average_values = {}
                    max_values = {}
                    for vv, radius_var in enumerate(list(radius_var_dict.keys())):

                        #convert perimeter points/extreme points from geographic to mollweide coordinates
                        perimeter_points = np.array(copy.deepcopy(unique_pair['points_{}'.format(radius_var_dict[radius_var])]))
                        perimeter_points = mollweide_crs.transform_points(geographic_crs, perimeter_points[:,0], perimeter_points[:,1])[:,:2]                 
                        perimeter_extreme_points = copy.deepcopy(unique_pair['extreme_{}'.format(radius_var_dict[radius_var])])
                        perimeter_extreme_points['N'] = mollweide_crs.transform_point(current_longitude, perimeter_extreme_points['N'], geographic_crs)[1] 
                        perimeter_extreme_points['E'] = mollweide_crs.transform_point(perimeter_extreme_points['E'], current_latitude, geographic_crs)[0] 
                        perimeter_extreme_points['S'] = mollweide_crs.transform_point(current_longitude, perimeter_extreme_points['S'], geographic_crs)[1] 
                        perimeter_extreme_points['W'] = mollweide_crs.transform_point(perimeter_extreme_points['W'], current_latitude, geographic_crs)[0] 

                        #get relevant indices of model which intersect with perimeter points
                        relevant_grid_indices = get_relevant_gridbox_indices(perimeter_points,perimeter_extreme_points,grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats,projection_crs=mollweide_crs)

                        relevant_grid_values = []
                        relevant_grid_weights = []
                        for box_indices in relevant_grid_indices:
                            relevant_gridbox_value = int(classes[box_indices[1],box_indices[0]].item())
                            relevant_grid_values.append(relevant_gridbox_value)
                            relevant_grid_weights.append(box_indices[2])

                        #dataset provides classification?
                        if dataset == 'GHS_SMOD_POPMT':
                            #get weighted mode of classifications from all gridcells intersecting with perimeter, using the fraction of gridcells inside perimeter as weights 
                            #if there is a tie, the lowest code is kept preferentially 
                            modal_codes[radius_var] = weighted_mode(relevant_grid_values, relevant_grid_weights)

                        #dataset provides numeric value
                        else:
                            #get weighted average in relevant gridcells     
                            average_values[radius_var] = np.average(relevant_grid_values, weights=relevant_grid_weights)
                            #get variable for max in radius
                            max_radius_var = radius_var.replace('average','max')
                            #get max value in relevant gridcells
                            max_values[max_radius_var] = np.max(relevant_grid_values)
                    
                #dataset provides classification?
                if dataset == 'GHS_SMOD_POPMT':
                    #set gridcell classifications (as strings associated with codes)
                    self.metadata[relevant_metadata_fields[0]][metadata_index] = GHSL_definitions[gridcell_class_code]
                    for vv, radius_var in enumerate(list(radius_var_dict.keys())):
                        #set 'modal_5km' and 'modal_25km' classifications
                        modal_code = modal_codes[radius_var]
                        self.metadata[radius_var][metadata_index] = GHSL_definitions[modal_code]
                #dataset provides numeric value
                else:
                    #set gridcell values
                    self.metadata[relevant_metadata_fields[0]][metadata_index] = gridcell_value
                    for vv, radius_var in enumerate(list(radius_var_dict.keys())):
                        #set 'average_5km' and 'average_25km' value 
                        self.metadata[radius_var][metadata_index] = average_values[radius_var]

                        #get variable for max in radius
                        max_radius_var = radius_var.replace('average','max')
                        #set 'max_5km' and 'max_25km' value 
                        self.metadata[max_radius_var][metadata_index] = max_values[max_radius_var]

                #set current longitude/latitude as previous values
                previous_longitude = copy.deepcopy(current_longitude)
                previous_latitude = copy.deepcopy(current_latitude)

                #set current class file to be relevant class file
                current_class_file = copy.deepcopy(relevant_class_file)    
     

    def get_NOAA_DMSP_OLS(self): 

        '''get NOAA DMSP OLS nighttime stable lights'''

        self.run_details['log_file_str'] += 'GETTING NOAA DMSP OLS NIGHTTIME STABLE LIGHTS\n'

        #get all year files for specific dataset
        class_files = sorted(glob.glob(self.run_details['data_root']+'globally_gridded_metadata/NOAA_DMSP-OLS_v4/NOAA_DMSP-OLS_V4*'))
        #get years of all classification dataset files --> convert to datetimes (in midpoint of year --> July 2nd)
        class_years = [int(class_file.split('nighttime_lights_')[-1].split('.nc')[0]) for class_file in class_files]
        #convert years to datetimes (in midpoint of year --> July 2nd 12:00 for non-leap years, July 2nd 00:00 for leap years)
        class_datetimes = [datetime.datetime(year, 7, 2, 0, 0, 0) if calendar.isleap(year) else datetime.datetime(year, 7, 2, 12, 0, 0) for year in class_years]        
        #convert datetimes to minutes since reference
        class_times = np.array([datetime_to_fractional_minutes_from_reference(class_datetime) for class_datetime in class_datetimes])
            
        #set relevant metadata fields for function
        relevant_metadata_fields = ['NOAA-DMSP-OLS_v4_nighttime_stable_lights', 'NOAA-DMSP-OLS_v4_average_nighttime_stable_lights_5km', 'NOAA-DMSP-OLS_v4_average_nighttime_stable_lights_25km', 'NOAA-DMSP-OLS_v4_max_nighttime_stable_lights_5km', 'NOAA-DMSP-OLS_v4_max_nighttime_stable_lights_25km']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #define radiuses of average metadata classifications that will be calculated
        average_radius_var_dict = {'NOAA-DMSP-OLS_v4_average_nighttime_stable_lights_5km':5.0, 'NOAA-DMSP-OLS_v4_average_nighttime_stable_lights_25km':25.0}

        #define current class file in memory
        current_class_file = ''

        #initialise previous longitude/latitude
        previous_longitude = -999.9
        previous_latitude = -999.9

        #iterate through all metadata columns
        for metadata_index, metadata_column in enumerate(self.metadata):

            #get current longitude/latitude
            current_longitude = metadata_column['longitude']
            current_latitude = metadata_column['latitude']
            #get unique pair dictionary for longitude/latitude pair
            for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():
                if (unique_pair['longitude'] == current_longitude) & (unique_pair['latitude'] == current_latitude):
                    break

            #get the midpoint of the metadata window
            if metadata_index != (len(self.metadata['latitude']) - 1):
                metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + ((self.metadata['meta_update_stamp'][metadata_index+1]-self.metadata['meta_update_stamp'][metadata_index])/2.)
            else:
                metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + (((self.data['time'][-1] + self.data['temporal_resolution'][-1])-self.metadata['meta_update_stamp'][metadata_index])/2.)    

            #find which data file has data closest to the metadata midpoint
            relevant_class_file = class_files[np.abs(class_times - metadata_midpoint).argmin()]
            
            #if relevant class file is not the same as the one in memory, then read new file 
            if relevant_class_file != current_class_file:
    
                #read class file frame (indexed by: latitude, longitude)
                loaded_classes = Dataset(relevant_class_file)
                loaded_classes.set_auto_mask(False)

                #get lon/lat centres/edges
                grid_edge_lons = loaded_classes['lon_e'][:]
                grid_edge_lats = loaded_classes['lat_e'][:]
                grid_centre_lons = loaded_classes['lon'][:]
                grid_centre_lats = loaded_classes['lat'][:]

                #read data variable frame
                classes = loaded_classes['NL'] 

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue  

            #if class file is not the same, or longitude or latitude have changed then 
            #get gridcell indices that longitude/latitude coordinates are contained in  
            if (relevant_class_file != current_class_file) or (previous_longitude != current_longitude) or (previous_latitude != current_latitude):
                lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

                #get gridcell specific value
                gridcell_value = classes[lat_i,lon_i].item()

                #get the average/max values from the gridcells fractions which intersect in a radius of 5km/25km around the station location
                #initialise dictionaries for storing average/max values
                average_values = {}
                max_values = {}
                for vv, average_radius_var in enumerate(list(average_radius_var_dict.keys())):

                    #get relevant indices of model which intersect with perimeter points
                    relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_{}'.format(average_radius_var_dict[average_radius_var])],unique_pair['extreme_{}'.format(average_radius_var_dict[average_radius_var])],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
                    relevant_grid_values = []
                    relevant_grid_weights = []
                    for box_indices in relevant_grid_indices:
                        relevant_gridbox_value = classes[box_indices[1],box_indices[0]].item()
                        #do not include unclassified 255 values as valid values
                        if relevant_gridbox_value != 255:
                            relevant_grid_values.append(relevant_gridbox_value)
                            relevant_grid_weights.append(box_indices[2])

                    #get weighted average in relevant gridcells (force to be integer)
                    if len(relevant_grid_values) > 0:  
                        average_values[average_radius_var] = np.around(np.average(relevant_grid_values, weights=relevant_grid_weights))
                    else:
                        average_values[average_radius_var] = 255

                    #get variable for max in radius
                    max_radius_var = average_radius_var.replace('average','max')
                    #get max value in relevant gridcells
                    if len(relevant_grid_values) > 0:
                        max_values[max_radius_var] = np.max(relevant_grid_values)
                    else:
                        max_values[max_radius_var] = 255

            #set gridcell values
            #if gridcell value == 255, then it is "unclassified" as have no cloud-free observations, so it should be NaN, so continue with metadata field unassigned  
            if gridcell_value != 255:
                self.metadata['NOAA-DMSP-OLS_v4_nighttime_stable_lights'][metadata_index] = gridcell_value
            for vv, average_radius_var in enumerate(list(average_radius_var_dict.keys())):
                #set 'average_5km' and 'average_25km' value --> do not set if average value is 255
                if average_values[average_radius_var] != 255:
                    self.metadata[average_radius_var][metadata_index] = average_values[average_radius_var]

                #get variable for max in radius
                max_radius_var = average_radius_var.replace('average','max')
                #set 'max_5km' and 'max_25km' --> do not set if max value is 255
                if max_values[max_radius_var] != 255:
                    self.metadata[max_radius_var][metadata_index] = max_values[max_radius_var]

            #set current longitude/latitude as previous values
            previous_longitude = copy.deepcopy(current_longitude)
            previous_latitude = copy.deepcopy(current_latitude)

            #set current class file to be relevant class file
            current_class_file = copy.deepcopy(relevant_class_file)


    def get_GPW_population_density(self): 

        '''get GPW population density'''

        self.run_details['log_file_str'] += 'GETTING GPW POPULATION DENSITY\n'

        #get all year files for specific dataset
        class_files = sorted(glob.glob(self.run_details['data_root']+'globally_gridded_metadata/GPW_POPULATION_DENSITY/GPW*'))
        #get years of all classification dataset files 
        class_years = [int(class_file.split('POPULATION_DENSITY_')[-1].split('.nc')[0]) for class_file in class_files]
        #convert years to datetimes (in midpoint of year --> July 2nd 12:00 for non-leap years, July 2nd 00:00 for leap years)
        class_datetimes = [datetime.datetime(year, 7, 2, 0, 0, 0) if calendar.isleap(year) else datetime.datetime(year, 7, 2, 12, 0, 0) for year in class_years]    
        #convert datetimes to minutes since reference
        class_times = np.array([datetime_to_fractional_minutes_from_reference(class_datetime) for class_datetime in class_datetimes])
            
        #set relevant metadata fields for function
        relevant_metadata_fields = ['GPW_population_density', 'GPW_average_population_density_5km', 'GPW_average_population_density_25km', 'GPW_max_population_density_5km', 'GPW_max_population_density_25km']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #define radiuses of average metadata classifications that will be calculated
        average_radius_var_dict = {'GPW_average_population_density_5km':5.0, 'GPW_average_population_density_25km':25.0}

        #define current class file in memory
        current_class_file = ''

        #initialise previous longitude/latitude
        previous_longitude = -999.9
        previous_latitude = -999.9

        #iterate through all metadata columns
        for metadata_index, metadata_column in enumerate(self.metadata):

            #get current longitude/latitude
            current_longitude = metadata_column['longitude']
            current_latitude = metadata_column['latitude']
            #get unique pair dictionary for longitude/latitude pair
            for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():
                if (unique_pair['longitude'] == current_longitude) & (unique_pair['latitude'] == current_latitude):
                    break

            #get the midpoint of the metadata window
            if metadata_index != (len(self.metadata['latitude']) - 1):
                metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + ((self.metadata['meta_update_stamp'][metadata_index+1]-self.metadata['meta_update_stamp'][metadata_index])/2.)
            else:
                metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + (((self.data['time'][-1] + self.data['temporal_resolution'][-1])-self.metadata['meta_update_stamp'][metadata_index])/2.)    

            #find which data file has data closest to the metadata midpoint
            relevant_class_file = class_files[np.abs(class_times - metadata_midpoint).argmin()]
            
            #if relevant class file is not the same as the one in memory, then read new file 
            if relevant_class_file != current_class_file:

                #read class file frame (indexed by: latitude, longitude)
                loaded_classes = Dataset(relevant_class_file)
                loaded_classes.set_auto_mask(False)

                #get lon/lat centres/edges
                grid_edge_lons = loaded_classes['lon_e'][:]
                grid_edge_lats = loaded_classes['lat_e'][:]
                grid_centre_lons = loaded_classes['lon'][:]
                grid_centre_lats = loaded_classes['lat'][:]

                #read data variable frame
                classes = loaded_classes['PD']   

            #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
            if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                continue  

            #if class file is not the same, or longitude or latitude have changed then 
            #get gridcell indices that longitude/latitude coordinates are contained in  
            if (relevant_class_file != current_class_file) or (previous_longitude != current_longitude) or (previous_latitude != current_latitude):
                lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

                #get gridcell specific value
                gridcell_value = classes[lat_i,lon_i].item()

                #get the average/max values from the gridcells fractions which intersect in a radius of 5km/25km around the station location
                #initialise dictionaries for storing average/max values
                average_values = {}
                max_values = {}
                for vv, average_radius_var in enumerate(list(average_radius_var_dict.keys())):

                    #get relevant indices of model which intersect with perimeter points
                    relevant_grid_indices = get_relevant_gridbox_indices(unique_pair['points_{}'.format(average_radius_var_dict[average_radius_var])],unique_pair['extreme_{}'.format(average_radius_var_dict[average_radius_var])],grid_centre_lons,grid_edge_lons,grid_centre_lats,grid_edge_lats)
                    relevant_grid_values = []
                    relevant_grid_weights = []
                    for box_indices in relevant_grid_indices:
                        relevant_gridbox_value = classes[box_indices[1],box_indices[0]].item()
                        relevant_grid_values.append(relevant_gridbox_value)
                        relevant_grid_weights.append(box_indices[2])

                    #get weighted average in relevant gridcells     
                    average_values[average_radius_var] = np.average(relevant_grid_values, weights=relevant_grid_weights)

                    #get variable for max in radius
                    max_radius_var = average_radius_var.replace('average','max')
                    #get max value in relevant gridcells
                    max_values[max_radius_var] = np.max(relevant_grid_values)

            #set gridcell values
            self.metadata['GPW_population_density'][metadata_index] = gridcell_value
            for vv, average_radius_var in enumerate(list(average_radius_var_dict.keys())):
                #set 'average_5km' and 'average_25km' value 
                self.metadata[average_radius_var][metadata_index] = average_values[average_radius_var]

                #get variable for max in radius
                max_radius_var = average_radius_var.replace('average','max')
                #set 'max_5km' and 'max_25km' value 
                self.metadata[max_radius_var][metadata_index] = max_values[max_radius_var]

            #set current longitude/latitude as previous values
            previous_longitude = copy.deepcopy(current_longitude)
            previous_latitude = copy.deepcopy(current_latitude)

            #set current class file to be relevant class file
            current_class_file = copy.deepcopy(relevant_class_file)


    def get_OMI_NO2(self):

        '''get OMI NO2 column data'''

        self.run_details['log_file_str'] += 'GETTING OMI NO2 COLUMN DATA\n'

        #iterate through all OMI column NO2 variables, processing each one by one
        for dataset in ['OMI_level3_column_annual_average_NO2','OMI_level3_column_cloud_screened_annual_average_NO2','OMI_level3_tropospheric_column_annual_average_NO2','OMI_level3_tropospheric_column_cloud_screened_annual_average_NO2']:

            #get all year files for specific classification dataset
            class_files = sorted(glob.glob(self.run_details['data_root']+'globally_gridded_metadata/OMI_NO2/{}*'.format(dataset)))
            #get years of all classification dataset files 
            class_years = [int(class_file.split('_annual_average_NO2_')[-1].split('.nc')[0]) for class_file in class_files]
            #convert years to datetimes (in midpoint of year --> July 2nd 12:00 for non-leap years, July 2nd 00:00 for leap years)
            class_datetimes = [datetime.datetime(year, 7, 2, 0, 0, 0) if calendar.isleap(year) else datetime.datetime(year, 7, 2, 12, 0, 0) for year in class_years]
            #convert datetimes to minutes since reference
            class_times = np.array([datetime_to_fractional_minutes_from_reference(class_datetime) for class_datetime in class_datetimes])
            
            #set relevant metadata fields for function
            self.metadata_fields_to_update.append(dataset)

            #define current class file in memory
            current_class_file = ''

            #initialise previous longitude/latitude
            previous_longitude = -999.9
            previous_latitude = -999.9

            #iterate through all metadata columns
            for metadata_index, metadata_column in enumerate(self.metadata):

                #get current longitude/latitude
                current_longitude = metadata_column['longitude']
                current_latitude = metadata_column['latitude']

                #get the midpoint of the metadata window
                if metadata_index != (len(self.metadata['latitude']) - 1):
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + ((self.metadata['meta_update_stamp'][metadata_index+1]-self.metadata['meta_update_stamp'][metadata_index])/2.)
                else:
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + (((self.data['time'][-1] + self.data['temporal_resolution'][-1])-self.metadata['meta_update_stamp'][metadata_index])/2.)    

                #find which data file has data closest to the metadata midpoint
                relevant_class_file = class_files[np.abs(class_times - metadata_midpoint).argmin()]

                #if relevant class file is not the same as the one in memory, then read new file 
                if relevant_class_file != current_class_file:
    
                    #read class file frame (indexed by: latitude, longitude)
                    loaded_classes = Dataset(relevant_class_file)
                    loaded_classes.set_auto_mask(False)

                    #get lon/lat centres/edges
                    grid_edge_lons = loaded_classes['lon_e'][:]
                    grid_edge_lats = loaded_classes['lat_e'][:]
                    grid_centre_lons = loaded_classes['lon'][:]
                    grid_centre_lats = loaded_classes['lat'][:]

                    #read data variable frame
                    classes = loaded_classes['NO2']

                #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
                if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                    continue

                #if class file is not the same, or longitude or latitude have changed then 
                #get gridcell indices that longitude/latitude coordinates are contained in  
                if (relevant_class_file != current_class_file) or (previous_longitude != current_longitude) or (previous_latitude != current_latitude):
                    lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)
                
                #set gridcell value
                self.metadata[dataset][metadata_index] = classes[lat_i,lon_i].item()

                #set current longitude/latitude as previous values
                previous_longitude = copy.deepcopy(current_longitude)
                previous_latitude = copy.deepcopy(current_latitude)

                #set current class file to be relevant class file
                current_class_file = copy.deepcopy(relevant_class_file)


    def get_EDGAR_emissions(self):

        '''get EDGAR emissions data'''

        self.run_details['log_file_str'] += 'GETTING EDGAR EMISSIONS DATA\n'

        #iterate through all EDGAR emission species, processing each one by one
        datasets = ['EDGAR_v4.3.2_annual_average_BC_emissions', 'EDGAR_v4.3.2_annual_average_biogenic_PM2.5_emissions',
                    'EDGAR_v4.3.2_annual_average_CO_emissions', 'EDGAR_v4.3.2_annual_average_fossilfuel_PM2.5_emissions',
                    'EDGAR_v4.3.2_annual_average_NH3_emissions', 'EDGAR_v4.3.2_annual_average_NMVOC_emissions',    
                    'EDGAR_v4.3.2_annual_average_NOx_emissions', 'EDGAR_v4.3.2_annual_average_OC_emissions',
                    'EDGAR_v4.3.2_annual_average_PM10_emissions', 'EDGAR_v4.3.2_annual_average_SO2_emissions']

        for dataset in datasets: 

            #get all year files for specific classification dataset
            class_files = sorted(glob.glob(self.run_details['data_root']+'globally_gridded_metadata/EDGAR_v4.3.2/{}*'.format(dataset)))
            #get years of all classification dataset files 
            class_years = [int(class_file.split('_emissions_')[-1].split('.nc')[0]) for class_file in class_files]
            #convert years to datetimes (in midpoint of year --> July 2nd 12:00 for non-leap years, July 2nd 00:00 for leap years)
            class_datetimes = [datetime.datetime(year, 7, 2, 0, 0, 0) if calendar.isleap(year) else datetime.datetime(year, 7, 2, 12, 0, 0) for year in class_years]
            #convert datetimes to minutes since reference
            class_times = np.array([datetime_to_fractional_minutes_from_reference(class_datetime) for class_datetime in class_datetimes])
            
            #set relevant metadata fields for function
            self.metadata_fields_to_update.append(dataset)

            #define current class file in memory
            current_class_file = ''

            #initialise previous longitude/latitude
            previous_longitude = -999.9
            previous_latitude = -999.9

            #iterate through all metadata columns
            for metadata_index, metadata_column in enumerate(self.metadata):

                #get current longitude/latitude
                current_longitude = metadata_column['longitude']
                current_latitude = metadata_column['latitude']

                #get the midpoint of the metadata window
                if metadata_index != (len(self.metadata['latitude']) - 1):
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + ((self.metadata['meta_update_stamp'][metadata_index+1]-self.metadata['meta_update_stamp'][metadata_index])/2.)
                else:
                    metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + (((self.data['time'][-1] + self.data['temporal_resolution'][-1])-self.metadata['meta_update_stamp'][metadata_index])/2.)    

                #find which data file has data closest to the metadata midpoint
                relevant_class_file = class_files[np.abs(class_times - metadata_midpoint).argmin()]
            
                #if relevant class file is not the same as the one in memory, then read new file 
                if relevant_class_file != current_class_file:

                    #read class file frame (indexed by: latitude, longitude)
                    loaded_classes = Dataset(relevant_class_file)
                    loaded_classes.set_auto_mask(False)

                    #get lon/lat centres/edges
                    grid_edge_lons = loaded_classes['lon_e'][:]
                    grid_edge_lats = loaded_classes['lat_e'][:]
                    grid_centre_lons = loaded_classes['lon'][:]
                    grid_centre_lats = loaded_classes['lat'][:]

                    #get data_var name
                    all_vars = list(loaded_classes.variables.keys())
                    all_vars.remove('lon')
                    all_vars.remove('lon_e')
                    all_vars.remove('lat')
                    all_vars.remove('lat_e')
                    data_var = all_vars[0]

                    #read data variable frame
                    classes = loaded_classes[data_var]

                #if current latitude exceeds bounding range, then continue to next coordinate pair leaving current pair unclassified
                if (current_latitude < grid_edge_lats[0]) or (current_latitude > grid_edge_lats[-1]):
                    continue

                #if class file is not the same, or longitude or latitude have changed then 
                #get gridcell indices that longitude/latitude coordinates are contained in  
                if (relevant_class_file != current_class_file) or (previous_longitude != current_longitude) or (previous_latitude != current_latitude):
                    lon_i, lat_i = get_coordinate_gridcell_indices(current_longitude,current_latitude,grid_centre_lons,grid_centre_lats)

                #set gridcell value
                self.metadata[dataset][metadata_index] = classes[lat_i,lon_i].item()

                #set current longitude/latitude as previous values
                previous_longitude = copy.deepcopy(current_longitude)
                previous_latitude = copy.deepcopy(current_latitude)

                #set current class file to be relevant class file
                current_class_file = copy.deepcopy(relevant_class_file)


    def get_Joly_Peuch_classification_codes(self): 

        '''get Joly-Peuch classification codes'''

        self.run_details['log_file_str'] += 'GETTING JOLY-PEUCH CLASSIFICATION CODES\n'

        #first check if species processing is one which Joly-Peuch codes are produced for
        #if not, return from function early
        if self.run_details['parameter'] not in ['O3','NO2','SO2','CO','PM10','PM2.5']:
            return 

        #define dictionary which maps species to column names in Joly-Peuch classification files
        class_species_dict = {'O3':'o3', 'NO2':'no2', 'SO2':'so2', 'CO':'co', 'PM10':'pm10', 'PM2.5':'pm25'}

        #get raw station reference (without standard method info)
        raw_station_reference = self.metadata['station_reference'][0].split('_')[0]

        #get all year files for specific dataset
        class_files = sorted(glob.glob(self.run_details['data_root']+'globally_gridded_metadata/JOLY-PEUCH/JOLY-PEUCH*'))
        #get start/end years of all classification dataset files 
        class_start_years = [int(class_file.split('_CODES_')[-1].split('.npy')[0][:4]) for class_file in class_files]
        class_end_years = [int(class_file.split('_CODES_')[-1].split('.npy')[0][-4:]) for class_file in class_files]
        #convert midpoint datetimes between start/end years of files
        class_datetimes = [datetime.datetime(start_year, 1, 1, 0, 0) + ((datetime.datetime(end_year, 1, 1, 0, 0) - datetime.datetime(start_year, 1, 1, 0, 0))/2) for start_year,end_year in zip(class_start_years,class_end_years)]    
        #convert datetimes to minutes since reference
        class_times = np.array([datetime_to_fractional_minutes_from_reference(class_datetime) for class_datetime in class_datetimes])
            
        #set relevant metadata fields for function
        self.metadata_fields_to_update.append('Joly-Peuch_classification_code')

        #define current class file in memory
        current_class_file = ''

        #iterate through all metadata columns
        for metadata_index, metadata_column in enumerate(self.metadata):

            #get the midpoint of the current metadata window
            if metadata_index != (len(self.metadata['latitude']) - 1):
                metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + ((self.metadata['meta_update_stamp'][metadata_index+1]-self.metadata['meta_update_stamp'][metadata_index])/2.)
            else:
                metadata_midpoint = self.metadata['meta_update_stamp'][metadata_index] + (((self.data['time'][-1] + self.data['temporal_resolution'][-1])-self.metadata['meta_update_stamp'][metadata_index])/2.)    

            #find which data file has data closest to the metadata midpoint
            relevant_class_file = class_files[np.abs(class_times - metadata_midpoint).argmin()]

            #get associated networks
            associated_networks = self.metadata['associated_networks'][metadata_index]
            if (not pd.isnull(associated_networks)) & (associated_networks != ''):
                associated_network_codes = [pair.split(':')[1] for pair in associated_networks.split(';') if ':' in pair]            
            else:
                associated_network_codes = np.NaN

            #if relevant class file is not the same as the one in memory, then read new file 
            if relevant_class_file != current_class_file:

                #read class file 
                while True:
                    try:
                        loaded_classes = np.load(relevant_class_file, allow_pickle=True)
                    except:
                        time.sleep(10.0)
                        continue  
                    break

                #initialise class code as empty string
                class_code = ''

                #check if current raw station reference has a Joly-Peuch classification code
                if raw_station_reference in loaded_classes['code'][:]:

                    #get index for specific station
                    station_index = np.where(loaded_classes['code'][:] == raw_station_reference)[0][0]

                    #get specific class code for species/station
                    class_code = loaded_classes[class_species_dict[self.run_details['parameter']]][station_index].strip()

                #check if associated network code has a Joly-Peuch classification code
                elif type(associated_network_codes) == list:
                    associated_network_codes_bool = np.isin(associated_network_codes, loaded_classes['code'][:])
                    if associated_network_codes_bool.any():
                        associated_network_code = associated_network_codes[np.where(associated_network_codes_bool==True)[0][0]]
                
                        #get index for specific station
                        station_index = np.where(loaded_classes['code'][:] == associated_network_code)[0][0]

                        #get specific class code for species/station
                        class_code = loaded_classes[class_species_dict[self.run_details['parameter']]][station_index].strip()            

            #set classification code (if not empty string)
            if class_code != '':
                self.metadata['Joly-Peuch_classification_code'][metadata_index] = int(class_code)

            #set current class file to be relevant class file
            current_class_file = copy.deepcopy(relevant_class_file)


    def get_WWF_TEOW_classifications(self):

        '''get WWF TEOW terrestrial ecoregion/biome and realm classifications'''

        self.run_details['log_file_str'] += 'GETTING WWF TEOW CLASSIFICATION METADATA\n'

        #define all distinct WWF TEOW biogeographical realm classifications by code:string
        WWF_TEOW_realm_definitions = {
        'AA': 'Australasia',
        'AN': 'Antarctic',
        'AT': 'Afrotropics',
        'IM': 'IndoMalay',
        'NA': 'Nearctic',
        'NT': 'Neotropics',
        'OC': 'Oceania',
        'PA': 'Palearctic'
        }

        #define all distinct WWF TEOW biome classifications by code:string
        WWF_TEOW_biome_definitions = {
        1: 'Tropical and Subtropical Moist Broadleaf Forests',
        2: 'Tropical and Subtropical Dry Broadleaf Forests',
        3: 'Tropical and Subtropical Coniferous Forests',
        4: 'Temperate Broadleaf and Mixed Forests',
        5: 'Temperate Conifer Forests',
        6: 'Boreal Forests/Taiga',
        7: 'Tropical and Subtropical Grasslands, Savannas and Shrublands',
        8: 'Temperate Grasslands, Savannas and Shrublands',
        9: 'Flooded Grasslands and Savannas',
        10: 'Montane Grasslands and Shrublands',
        11: 'Tundra',
        12: 'Mediterranean Forests, Woodlands and Scrub',
        13: 'Deserts and Xeric Shrublands',
        14: 'Mangroves',
        }

        #load WWF TEOW shape file
        while True:
            try:
                WWF_TEOW_shape = gpd.read_file(self.run_details['data_root']+'globally_gridded_metadata/TEOW_WWF/wwf_terr_ecos.shp')
            except:
                time.sleep(10.0)
                continue
            break
        #set relevant metadata fields for function
        relevant_metadata_fields = ['WWF_TEOW_terrestrial_ecoregion', 'WWF_TEOW_biogeographical_realm', 'WWF_TEOW_biome']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #create Shapely Point from longitude and latitude pairing
            point = Point(current_longitude, current_latitude)

            #initialise variable for determining when a location has been placed within a polygon
            inside_a_polygon = False

            #iterate through all polygons, and check if lat/lon pairing is inside polygon
            for polygon_ii, polygon in enumerate(WWF_TEOW_shape['geometry']):

                #check if lon/lat pairing is inside/touching boundaries of polygon, if so assign associated metadata
                if (polygon.contains(point)) or (polygon.touches(point)):
    
                    #update variable now have placed point within polygon
                    inside_a_polygon = True
                
                    #get associated polygon metadata
                    ecoregion = WWF_TEOW_shape['ECO_NAME'][polygon_ii]
                    realm = WWF_TEOW_shape['REALM'][polygon_ii]
                    biome = WWF_TEOW_shape['BIOME'][polygon_ii]

                    #assign metadata
                    self.metadata['WWF_TEOW_terrestrial_ecoregion'][relevant_metadata_indices] = ecoregion
                    try:
                        self.metadata['WWF_TEOW_biogeographical_realm'][relevant_metadata_indices] = WWF_TEOW_realm_definitions[realm]                       
                    #if code not mapped, set class to be 'Water'
                    except:
                        self.metadata['WWF_TEOW_biogeographical_realm'][relevant_metadata_indices] = 'Water'
                    try:
                        self.metadata['WWF_TEOW_biome'][relevant_metadata_indices] = WWF_TEOW_biome_definitions[int(biome)] 
                    #if code not mapped, set class to be 'Water'
                    except:
                        self.metadata['WWF_TEOW_biome'][relevant_metadata_indices] = 'Water'
                    break
            
            #if not placed point within polygon, then set classes to be 'Water'
            if not inside_a_polygon:
                self.metadata['WWF_TEOW_terrestrial_ecoregion'][relevant_metadata_indices] = 'Water'
                self.metadata['WWF_TEOW_biogeographical_realm'][relevant_metadata_indices] = 'Water'
                self.metadata['WWF_TEOW_biome'][relevant_metadata_indices] = 'Water' 
    
    def get_NUTS_region(self):

        '''get NUTS socio-economic regions

        Levels:
        NUTS 0: national boundaries
        NUTS 1: major socio-economic regions
        NUTS 2: basic regions for the application of regional policies
        NUTS 3: small regions for specific diagnoses

        '''

        self.run_details['log_file_str'] += 'GETTING NUTS REGION METADATA\n'

        # #define all distinct WWF TEOW biogeographical realm classifications by code:string
        # WWF_TEOW_realm_definitions = {
        # 'AA': 'Australasia',
        # 'AN': 'Antarctic',
        # 'AT': 'Afrotropics',
        # 'IM': 'IndoMalay',
        # 'NA': 'Nearctic',
        # 'NT': 'Neotropics',
        # 'OC': 'Oceania',
        # 'PA': 'Palearctic'
        # }

        # #define all distinct WWF TEOW biome classifications by code:string
        # WWF_TEOW_biome_definitions = {
        # 1: 'Tropical and Subtropical Moist Broadleaf Forests',
        # 2: 'Tropical and Subtropical Dry Broadleaf Forests',
        # 3: 'Tropical and Subtropical Coniferous Forests',
        # 4: 'Temperate Broadleaf and Mixed Forests',
        # 5: 'Temperate Conifer Forests',
        # 6: 'Boreal Forests/Taiga',
        # 7: 'Tropical and Subtropical Grasslands, Savannas and Shrublands',
        # 8: 'Temperate Grasslands, Savannas and Shrublands',
        # 9: 'Flooded Grasslands and Savannas',
        # 10: 'Montane Grasslands and Shrublands',
        # 11: 'Tundra',
        # 12: 'Mediterranean Forests, Woodlands and Scrub',
        # 13: 'Deserts and Xeric Shrublands',
        # 14: 'Mangroves',
        # }

        #load WWF TEOW shape file
        while True:
            try:
                NUTS_shape = gpd.read_file(self.run_details['data_root']+'globally_gridded_metadata/NUTS/NUTS_RG_01M_2024_4326.shp')
            except:
                time.sleep(10.0)
                continue
            break

        #set relevant metadata fields for function
        relevant_metadata_fields = ['NUTS_name_level_1', 'NUTS_name_level_2', 'NUTS_name_level_3', 'NUTS_code_level_0', 'NUTS_code_level_1', 'NUTS_code_level_2', 'NUTS_code_level_3']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        # nuts levels
        nuts_levels = [1, 2, 3]
        for nuts_level in nuts_levels:

            # cut shapefile for each nuts level
            NUTS_shape = NUTS_shape[NUTS_shape['LEVL_CODE'] == str(nuts_level)]

            #iterate through all unique longitude/latitude pairs
            for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

                #get current unique longitude/latitude
                current_longitude = unique_pair['longitude']
                current_latitude = unique_pair['latitude']

                #get metadata indices associated with current unique longitude/latitude pair
                relevant_metadata_indices = unique_pair['metadata_indices']

                #create Shapely Point from longitude and latitude pairing
                point = Point(current_longitude, current_latitude)

                #initialise variable for determining when a location has been placed within a polygon
                inside_a_polygon = False

                #iterate through all polygons, and check if lat/lon pairing is inside polygon
                for polygon_ii, polygon in enumerate(NUTS_shape['geometry']):

                    #check if lon/lat pairing is inside/touching boundaries of polygon, if so assign associated metadata
                    if (polygon.contains(point)) or (polygon.touches(point)):
        
                        #update variable now have placed point within polygon
                        inside_a_polygon = True
                    
                        #get associated polygon metadata
                        nuts_country_code = NUTS_shape['CNTR_CODE'][polygon_ii]
                        nuts_region_info = NUTS_shape['NUTS_NAME'][polygon_ii]
                        nuts_ID = NUTS_shape['NUTS_ID'][polygon_ii]

                        #assign metadata
                        self.metadata['NUTS_code_level_0'][relevant_metadata_indices] = nuts_country_code
                        self.metadata['NUTS_name_level_'+str(nuts_level)][relevant_metadata_indices] = nuts_region_info
                        self.metadata['NUTS_code_level_'+str(nuts_level)][relevant_metadata_indices] = nuts_ID
                        
                        break
                
                #if not placed point within polygon, then set classes to be ''
                if not inside_a_polygon:
                    self.metadata['NUTS_code_level_'+str(nuts_level)][relevant_metadata_indices] = ''
                    self.metadata['NUTS_name_level_'+str(nuts_level)][relevant_metadata_indices] = ''
                    self.metadata['NUTS_code_level_'+str(nuts_level)][relevant_metadata_indices] = ''


    def get_WMO_region(self):

        '''determine which WMO region a lon/lat pairing is contained within (done using Shapely)'''

        self.run_details['log_file_str'] += 'GETTING WMO REGION METADATA\n'

        #get WMO region geojson data
        wmo_data = pd.read_json(self.run_details['data_root']+'globally_gridded_metadata/WMO/wmo.geojson')

        #get MultiPolygon boundaries for each WMO region
        #Region 1: Africa
        region1 = shape(wmo_data['features'][0]['geometry'])
        #Region 2: Asia
        region2 = shape(wmo_data['features'][1]['geometry'])
        #Region 3: South America
        region3 = shape(wmo_data['features'][2]['geometry'])
        #Region 4: North America, Central America and the Caribbean
        region4 = shape(wmo_data['features'][3]['geometry'])
        #Region 5: South-West Pacific
        region5 = shape(wmo_data['features'][4]['geometry'])
        #Region 6: Europe
        region6 = shape(wmo_data['features'][5]['geometry'])
        #Region 7: Antarctic
        region7 = MultiPolygon([Polygon([[-180,-60],[-180,-90],[180,-90],[180,-60]])])
        
        all_regions = [region1,region2,region3,region4,region5,region6,region7]

        #dictionary to return region by string from defined code
        code_to_string_dict = {1:'Africa', 2:'Asia', 3:'South America', 4:'North America, Central America and the Caribbean', 5:'South-West Pacific', 6:'Europe', 7:'Antarctic'}

        #set relevant metadata fields for function
        relevant_metadata_fields = ['WMO_region']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #create Shapely Point from longitude and latitude pairing
            point = Point(current_longitude, current_latitude)

            #iterate through each region multipolygon to determine which region the lon/lat pairing is contained within 
            for i, region in enumerate(all_regions):
                #iterate through polygons within multipolygon 
                for region_polygon in region.geoms:
                    #check if lon/lat pairing is inside/touching boundaries of polygon
                    if (region_polygon.contains(point)) or (region_polygon.touches(point)):
                        #get which number WMO region the lon/lat pairing is contained within.
                        WMO_region_code = i+1
                        #get string for WMO region code
                        WMO_region = code_to_string_dict[WMO_region_code]
                        #set WMO region
                        self.metadata['WMO_region'][relevant_metadata_indices] = WMO_region
                        break

    def get_reverse_geocode_station_information(self):

        '''get reverse geocode station information for all unique lon/lat pairs
           fill all fields where there is no prior information'''

        self.run_details['log_file_str'] += 'GETTING REVERSE GEOCODE STATION METADATA\n'

        #set relevant metadata fields for function
        relevant_metadata_fields = ['city','administrative_country_division_1','administrative_country_division_2','country']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get reverse geocode station information
            #add to relevant meta fields (if do not already have prior information)
            reverse_geocode_dict = reverse_geocode_station_information(longitude=current_longitude, latitude=current_latitude)
            #iterate though returned geocode station information fields
            for geo_key in list(reverse_geocode_dict.keys()):
                #iterate through all relevant metadata indices
                for relevant_metadata_index in relevant_metadata_indices:
                    #have no current valid information?
                    if (pd.isnull(self.metadata[geo_key][relevant_metadata_index])) or (self.metadata[geo_key][relevant_metadata_index] == ''):
                        #set field
                        self.metadata[geo_key][relevant_metadata_index] = reverse_geocode_dict[geo_key]     
 

    def get_station_local_timezone(self):

        '''get station local timezone string for all unique lon/lat pairs'''

        self.run_details['log_file_str'] += 'GETTING STATION LOCAL TIMEZONE METADATA\n'

        #set relevant metadata fields for function
        relevant_metadata_fields = ['station_timezone']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        #iterate through all unique longitude/latitude pairs
        for unique_pair_ii, unique_pair in self.unique_pair_dictionary.items():

            #get current unique longitude/latitude
            current_longitude = unique_pair['longitude']
            current_latitude = unique_pair['latitude']

            #get metadata indices associated with current unique longitude/latitude pair
            relevant_metadata_indices = unique_pair['metadata_indices']

            #get station local timezone
            self.metadata['station_timezone'][relevant_metadata_indices] = local_timezone_name(longitude=current_longitude, latitude=current_latitude)        


    def get_WIGOS_station_identifier(self):

        '''get WIGOS station identifier from station_reference / associated networks'''

        self.run_details['log_file_str'] += 'GETTING WIGOS STATION IDENTIFIER METADATA\n'

        #set relevant metadata fields for function
        relevant_metadata_fields = ['WIGOS_station_identifier']
        self.metadata_fields_to_update.extend(relevant_metadata_fields)

        # load WIGOS_stations_per_network
        WIGOS_stations_per_network = np.load(self.run_details['data_root']+'globally_gridded_metadata/WIGOS/WIGOS_stations_per_network.npy', allow_pickle=True)
        WIGOS_stations_per_network = WIGOS_stations_per_network.tolist()
        #some station references are separated by '/'. Separate these out.
        for network in WIGOS_stations_per_network.keys():
            stations = list(WIGOS_stations_per_network[network].keys())
            for station in stations:
                if '/' in station:
                    station_split = station.split('/')
                    WIGOS_stations_per_network[network][station_split[0]] = WIGOS_stations_per_network[network][station]
                    WIGOS_stations_per_network[network][station_split[1]] = WIGOS_stations_per_network[network][station]

        # define mapping for GHOST networks to WIGOS
        GHOSTnetworks_to_WIGOS_mapping = {
        'AERONET_v3_lev1.5':{'AERONET':'remove_'},
        'AERONET_v3_lev2.0':{'AERONET':'remove_'},
        'CAPMoN':           {'CAPMoN':'GHOSTlast3'},
        'EANET':            {'EANET':'bystationname'},
        'EBAS':             {'EMEP':''},
        'US_EPA_CASTNET':   {'CASTNET':''},
        'US_NADP_AMNet':    {'NADP':'first4'},
        'US_NADP_AMoN':     {'NADP':'first4'},
        'WMO_WDCGG':        {'GAW Global':'', 'GAW Regional':'', 'GAW Local':''}
        }

        # define mapping for GHOST associated networks to WIGOS
        GHOSTassociatednetworks_to_WIGOS_mapping = {
        'AERONET': {'AERONET':'remove_'},
        'EARLINET':{'EARLINET':''},
        'EBAS':    {'EMEP':''},
        'GAW':     {'GAW Global':'', 'GAW Regional':'', 'GAW Local':''},
        'ICOS':    {'ICOS':''},
        'NADP':    {'NADP':'first4'},
        'NDACC':   {'NDACC':'first4'},
        'WDCA':    {'GAW Global':'last3', 'GAW Regional':'last3', 'GAW Local':'last3'}
        }

        # define mapping of GHOST EANET station names to WIGOS EANET station names
        EANETGHOST_to_EANETWMO_mapping = {
        'bangkok':       'bangna bangkok',
        'kototabang':    'bukit kototabang',
        'ochiishi':      'cape ochiishi',
        'hedo':          'hedo (okinawa)',
        'mt. sto. tomas':'mt. st. thomas'
        }

        # try mapping GHOST network station_reference to WIGOS
        # if cannot then try by associated networks
        if self.run_details['network'] in GHOSTnetworks_to_WIGOS_mapping:

            # get list of networks in networks in WIGOS associated with network processing
            WIGOS_networks = list(GHOSTnetworks_to_WIGOS_mapping[self.run_details['network']].keys())
            for WIGOS_network in WIGOS_networks:

                # get station references across time record
                # also set to lower case and remove method 
                station_references = ['_'.join(ref.lower().split('_')[:-1]) for ref in self.metadata['station_reference'][:]]

                # get station names across time record
                station_names = self.metadata['station_name'][:]

                # get WIGOS network spcific station references
                WIGOS_network_station_references = list(WIGOS_stations_per_network[WIGOS_network].keys())

                # set WIGOS_network_station_references to lower case and associate WIGOS identifiers
                WIGOS_network_station_references = {ref.lower():WIGOS_stations_per_network[WIGOS_network][ref]['wsi'] for ref in WIGOS_network_station_references}

                #get modifier fmt for WIGOS_network_station_references to match format of GHOST station_references (or vice versa)
                modifier_fmt = GHOSTnetworks_to_WIGOS_mapping[self.run_details['network']][WIGOS_network]
                if modifier_fmt == 'remove_':
                    WIGOS_network_station_references = {ref.replace('_',''):wig for ref,wig in WIGOS_network_station_references.items()}
                elif modifier_fmt == 'first4':
                    WIGOS_network_station_references = {ref[:4]:wig for ref,wig in WIGOS_network_station_references.items()}
                elif modifier_fmt == 'last3':
                    WIGOS_network_station_references = {ref[-3:]:wig for ref,wig in WIGOS_network_station_references.items()}
                elif modifier_fmt == 'GHOSTlast3':
                    station_references = [ref[-3:] for ref in station_references]
                elif (modifier_fmt == 'bystationname') & (self.run_details['network'] == 'EANET'):
                    station_references = [ref.lower() for ref in station_names]
                    WIGOS_network_station_references = {WIGOS_stations_per_network[WIGOS_network][ref]['station_name'].lower():WIGOS_stations_per_network[WIGOS_network][ref]['wsi'] for ref in WIGOS_network_station_references}
                    # alter station references to match WIGOS station names
                    for eanet_GHOST, eanet_WMO in EANETGHOST_to_EANETWMO_mapping.items():
                        if eanet_GHOST in station_references:
                            station_references[station_references.index(eanet_GHOST)] = eanet_WMO

                #iterate through unique station references / names
                for station_reference in np.unique(station_references):

                    # get indices associated with unique station reference / name
                    relevant_metadata_indices = np.where(np.array(station_references) == station_reference)[0]

                    #if match with WIGOS has already been found then continue to next station
                    if not pd.isnull(self.metadata['WIGOS_station_identifier'][relevant_metadata_indices[0]]):
                        continue

                    #otherwise, try mapping GHOST network station_reference to WIGOS
                    if station_reference in WIGOS_network_station_references:
                        self.metadata['WIGOS_station_identifier'][relevant_metadata_indices] = WIGOS_network_station_references[station_reference]

        # get associated networks acoss time record
        associated_networks = self.metadata['associated_networks'][:]

        # try matching non-matched stations to WIGOS by associated networks
        for station_associated_networks in np.unique(associated_networks):

            # if associated network is NaN, then continue     
            if pd.isnull(station_associated_networks):
                continue    

            # get indices associated with unique associated networks
            relevant_metadata_indices = np.where(associated_networks == station_associated_networks)[0]

            #if match with WIGOS has already been found then continue to next station
            if not pd.isnull(self.metadata['WIGOS_station_identifier'][relevant_metadata_indices[0]]):
                continue  

            # otherwise split associated networks
            station_associated_networks_split = station_associated_networks.split(';')
            for station_associated_network_ns in station_associated_networks_split:

                #if WIGOS match has been found in loop, break out of it
                if not pd.isnull(self.metadata['WIGOS_station_identifier'][relevant_metadata_indices[0]]):
                    break                     
                
                # get associated network and reference pair
                station_associated_network_ns_split = station_associated_network_ns.split(':')
                station_associated_network = station_associated_network_ns_split[0]
                # if associated is WIGOS, then set reference directly and break out of loop
                if station_associated_network == 'WIGOS':
                    self.metadata['WIGOS_station_identifier'][relevant_metadata_indices] = station_associated_network_ns_split[1]
                    break
                station_associated_network_reference = station_associated_network_ns_split[1].lower()

                if station_associated_network in GHOSTassociatednetworks_to_WIGOS_mapping:
                    WIGOS_associated_networks = list(GHOSTassociatednetworks_to_WIGOS_mapping[station_associated_network].keys())
                    for WIGOS_associated_network in WIGOS_associated_networks:

                        #if match with WIGOS has already been found in lopp, break out of it
                        if not pd.isnull(self.metadata['WIGOS_station_identifier'][relevant_metadata_indices[0]]):
                            break

                        # get WIGOS network spcific station references
                        WIGOS_associated_network_station_references = list(WIGOS_stations_per_network[WIGOS_associated_network].keys())
                        
                        # set WIGOS_associated_network_station_references to lower case
                        # and associate WIGOS identifiers
                        WIGOS_associated_network_station_references = {ref.lower():WIGOS_stations_per_network[WIGOS_associated_network][ref]['wsi'] for ref in WIGOS_associated_network_station_references}

                        #get modifier fmt for WIGOS_network_station_references to match format of GHOST station_references (or vice versa)
                        modifier_fmt = GHOSTassociatednetworks_to_WIGOS_mapping[station_associated_network][WIGOS_associated_network]
                        if modifier_fmt == 'remove_':
                            WIGOS_associated_network_station_references = {ref.replace('_',''):wig for ref,wig in WIGOS_associated_network_station_references.items()}
                        elif modifier_fmt == 'first4':
                            WIGOS_associated_network_station_references = {ref[:4]:wig for ref,wig in WIGOS_associated_network_station_references.items()}
                        elif modifier_fmt == 'last3':
                            WIGOS_associated_network_station_references = {ref[-3:]:wig for ref,wig in WIGOS_associated_network_station_references.items()}

                        # check if station_reference in WIGOS_associated_network_station_references
                        if station_associated_network_reference in WIGOS_associated_network_station_references:
                            self.metadata['WIGOS_station_identifier'][relevant_metadata_indices] = WIGOS_associated_network_station_references[station_associated_network_reference]