import requests
import time
from datetime import date
from datetime import timedelta
import datetime
import pandas as pd
import os.path
import time
import os
import csv
import json


def json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id):
    # does the job

    identifier = meta_params['identifier']

    if meta_basefile == True:  # create json from base metadata file
        network_provided_meta_file = '{}/{}/metadata/{}/network_provided/{}_META_{}.csv'.format(dl_root, network, version, network, meta_params['base_file_id'])
        json_base_file = create_json_file(dl_root, network, version, network_provided_meta_file, meta_params)

        # safe base file
        with open('{}/{}/metadata/{}/processed/{}_META.json'.format(dl_root, network, version, network), 'w', encoding=meta_params['encoding']) as f:
            f.write(json.dumps(json_base_file, indent=4))

    else:
        pass

    if meta_history == True: # parse all ever downloaded meta files
        # list all meta files
        meta_dir = '{}/{}/metadata/{}/network_provided/'.format(dl_root, network, version)
        network_provided_meta_files = [meta_dir+f for f in os.listdir(meta_dir) if (('META_' in f) and ('.csv' in f))]
        network_provided_meta_files = sorted(network_provided_meta_files)  # sort in right order
        
        for network_provided_meta_file in network_provided_meta_files: # loop through all files
            json_metadata_now = create_json_file(dl_root, network, version, network_provided_meta_file, meta_params)

            # compare current file with json basefile and update it if necessary
            compare_jsons(dl_root, network, version, json_metadata_now, meta_params)

    
    else: # only parse latest downloaded meta file
        # create json from current metadata file
        network_provided_present_meta_file = '{}/{}/metadata/{}/network_provided/{}_META_{}.csv'.format(dl_root, network, version, network, present_meta_file_id)
        json_metadata_now = create_json_file(dl_root, network, version, network_provided_present_meta_file, meta_params)

        # compare current file with json basefile and update it if necessary
        compare_jsons(dl_root, network, version, json_metadata_now, meta_params)



def create_json_file(dl_root, network, version, network_provided_meta_file, meta_params):

    identifier = meta_params['identifier']
    delimiter_csv = meta_params['delimiter_meta_csv']

    # get update time of this json file
    if meta_params['network_provided_update_time'] == True:
        update_date = 'station_dependent'
    else: # use the time of download
        raw_date = network_provided_meta_file.split('META_')[-1].split('.csv')[0]
        if len(raw_date) == 4: # format YYYY
            raw_date = raw_date+'0101' # assume January 1st

        update_date = datetime.datetime.strptime(raw_date, '%Y%m%d')
        update_date = update_date.astimezone(datetime.timezone.utc).strftime('%Y-%m-%d')
        

    json_metadata = {}
    with open(network_provided_meta_file, 'r', encoding=meta_params['encoding']) as file:
        csv_filedata = csv.DictReader(file, delimiter=delimiter_csv)

        for row in csv_filedata:

            # find key
            if identifier == 'create_CAPMoN_identifier':
                key = row['ID']+'_'+row['Measurements_Mesures'].replace(' ', '')[:3]
            elif identifier == 'create_NOAA_ISD_identifier':
                key = row['USAF']+'_'+row['WBAN']
            elif identifier == 'create_NZ_ECAN_identifier':
                key = row['StationCode']+'_'+row['MonitorChannel']
            #elif identifier == 'create_EEA_AQ_eReporting_identifier':
            #    key = row['Air Quality Station EoI Code']+'_'+row['Sampling Point Id']+'_'+row['Process Id']
            elif identifier == 'create_EEA_AIRBASE_identifier':
                key = row['station_european_code']+'_'+row['component_caption']
            elif identifier == 'create_US_EPA_AQS_identifier':
                try: # monitors
                    key = row['State Code']+'_'+row['County Code']+'_'+row['Site Number']+'_'+row['Parameter Code']
                except: # sites
                    key = row['State Code']+'_'+row['County Code']+'_'+row['Site Number']
            else: # general case
                key = row[identifier]
            
            for parameter in row:
                if update_date == 'station_dependent': # create update date on the fly for certain networks
                    if network == 'CAPMoN':
                        update_date_fin = row['UPDATE_DATE'][:10] # works so far only for CASTNET
                    elif network == 'NAPMD':
                        update_date_fin = row['updated'][:10] # works so far only for CASTNET
                else:
                    update_date_fin = update_date

                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date_fin]} # create inner dictionary for every parameter
            json_metadata[key] = row


        # check for punctuation of lat, lon, alt, height
        for punct_variable in ['Latitude', 'Longitude', 'Altitude', 'Height']: # standard GHOST names
            if punct_variable in meta_params.keys():
                for network_spec_var_name in meta_params[punct_variable]: # loop through all possible variable names in one network, usually it should just be one name 
                    for station in json_metadata: # loop through all stations
                        if network_spec_var_name in json_metadata[station]: # clean punctuation discrepancies
                            input_val = json_metadata[station][network_spec_var_name]['values'][0] # check values
                            if input_val is not None:
                                input_val = input_val.replace(',', '.').replace(' ', '').replace('E', '').replace('N', '')

                                if input_val == '': # check if its empty string
                                    output_val = None
                                elif input_val == '-':  # check if its dash
                                    output_val = None
                                else:
                                    try:
                                        output_val = str(float(input_val))
                                    except: # if not convertible to float, disregard
                                        output_val = None

                                json_metadata[station][network_spec_var_name]['values'][0] = output_val
                            
                        else:
                            continue
            else:
                continue

    return json_metadata


def compare_jsons(dl_root, network, version, json_metadata_now, meta_params):

    # import json basefile
    if 'US_EPA_AQS' in network: # US_EPA_AQS special case: meta json is split by state
        # seperate information
        channel = network.split('-')[1]
        state_code = network.split('-')[2]
        network = network.split('-')[0]

        # open basefile per state and channel
        with open('{}/{}/metadata/{}/processed/{}/{}_{}_{}_META.json'.format(dl_root, network, version, channel, network, channel, state_code), 'r', encoding=meta_params['encoding']) as f:
            json_metadata = json.loads(f.read())

    elif 'EEA_AQ_eReporting' in network:  # EEA_AQ_eReporting special case: meta json is split by country
        state_code = network.split('-')[1]
        network = network.split('-')[0]

        # open basefile per country
        with open('{}/{}/metadata/{}/processed/{}_{}_META.json'.format(dl_root, network, version, network, state_code), 'r', encoding=meta_params['encoding']) as f:
            json_metadata = json.loads(f.read())

    else: # general case
        with open('{}/{}/metadata/{}/processed/{}_META.json'.format(dl_root, network, version, network), 'r', encoding=meta_params['encoding']) as f:
            json_metadata = json.loads(f.read())

    k=0 # number of abolished stations
    m=0 # number of new stations
    l=0 # number of new parameters per station
    station_abolished = []
    station_new = []  
    parameter_new = []

    # compare files
    for station in json_metadata: # loop through all the old stations
        if station in json_metadata_now.keys(): # if station is in current meta data, go on
            for parameter in json_metadata[station]:
                if parameter in json_metadata_now[station]: # check if parameter exists in new meta file
                    if ((json_metadata_now[station][parameter]['values'][0] == None) or (json_metadata_now[station][parameter]['values'][0] == '') or (json_metadata_now[station][parameter]['values'][0] == ' ')): 
                        # if new entry is NaN or empty string, disregard
                        continue
                    else: 
                        if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                            # if different value, append the standardised metadeta file
                            print("new {} --- old {}".format(json_metadata_now[station][parameter]['values'][0], json_metadata[station][parameter]['values'][-1]))
                            json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                            json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                        else:
                            pass
                else: # a parameter disappeared in the new file -> do nothing
                    pass 
        else:
            k=k+1
            station_abolished.append(station)


    for station in json_metadata_now: # loop through all the new stations
        if station in json_metadata.keys(): # if station is in old meta data
            pass # comparison was done before
        else: # new station appeared!
            m=m+1
            station_new.append(station)
            json_metadata.update({station: json_metadata_now[station]})

        for parameter in json_metadata_now[station]: # loop through all the parameters
            # is there a new parameter that wasn't in the old file?
            if parameter in json_metadata[station].keys():
                pass # parameter (column) is already there
            else:
                l=l+1
                parameter_new.append(parameter)
                json_metadata[station].update({parameter: json_metadata_now[station][parameter]})
        

    # summary
    print('================', network, '================')
    print('{} stations were abolished'.format(k))
    print('Abolished stations: {}'.format(station_abolished))
    print('{} new stations '.format(m))
    print('New stations: {}'.format(station_new))
    print('{} new parameters '.format(l))
    print('New parameters per station: {}'.format(parameter_new))


    # safe
    if 'US_EPA_AQS' in network: # US_EPA_AQS special case
        with open('{}/{}/metadata/{}/processed/{}/{}_{}_{}_META.json'.format(dl_root, network, version, channel, network, channel, state_code), 'w', encoding=meta_params['encoding']) as f:
            f.write(json.dumps(json_metadata, indent=4))

    elif 'EEA_AQ_eReporting' in network:
        with open('{}/{}/metadata/{}/processed/{}_{}_META.json'.format(dl_root, network, version, network, state_code), 'w', encoding=meta_params['encoding']) as f:
            f.write(json.dumps(json_metadata, indent=4))

    else: # general case
        with open('{}/{}/metadata/{}/processed/{}_META.json'.format(dl_root, network, version, network), 'w', encoding=meta_params['encoding']) as f:
            f.write(json.dumps(json_metadata, indent=4))