from bs4 import BeautifulSoup
import requests
import time
from datetime import date
from datetime import timedelta
import os.path
import pandas as pd
import numpy as np
import csv
import json


def download_data(mode, version, n_max_tries, max_time_per_dl):

    base_url = 'https://uk-air.defra.gov.uk/data/atom-dls/{}/{}/atom.en.xml'
    #base_url_til_1996 = 'https://uk-air.defra.gov.uk/data/atom-dls/observations/auto/GB_FixedObservations_{}_CRD.xml'
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    if mode == 'all':
        bdate = date(2022, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)
        #edate = date(1996, 1, 1) #date(1960, 1, 1) # date before record starts

        os.makedirs('/esarchive/obs/ghost/UK_AIR/original_files/{}/'.format(version), exist_ok=True)
        download_location = '/esarchive/obs/ghost/UK_AIR/original_files/{}/'.format(version)

    elif mode == 'nrt':
        version = mode
        bdate = date.today() 
        edate = date.today() + timedelta(days = 365)
        download_location = '/esarchive/obs/ghost/UK_AIR/original_files/nrt/'

    else:
        print('time mode inapplicable')


    # create date array, daily; format YYYYMMDD
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    print(years)
    download_feeds = ['auto', 'non-auto', 'aggregated']
   

    for download_feed in download_feeds:

        for year in years:

            os.makedirs('/esarchive/obs/ghost/UK_AIR/original_files/{}/{}/{}/'.format(version, download_feed, year), exist_ok=True)

            # get feed for each year and mode
            url = base_url.format(download_feed, year)
            r = requests.get(url, timeout=max_time_per_dl, headers=Headers)
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, features="html.parser")

                # find link to station feed
                station_feeds = soup.find_all(title="Feed containing the dataset in several formats")
                for station_feed in station_feeds:

                    if station_feed is not None:
                        station_feed_link = station_feed['href']

                        # request new page url
                        response = requests.get(station_feed_link, timeout=max_time_per_dl, headers=Headers)
                        if response.status_code == 200:
                            soup_dataset = BeautifulSoup(response.text, features="html.parser")
                            data_link = soup_dataset.find("link", type='application/gml+xml;charset=utf-8')['href']
                            print(data_link)
                            station = data_link.split('_')[-1]
                            station = station.split('.')[0]

                            # request download
                            res = requests.get(data_link, timeout=max_time_per_dl, headers=Headers)
                            if res.status_code == 200:

                                # check if xml contains any data
                                soup_data = BeautifulSoup(res.text, features="html.parser")
                                if soup_data.find("om:om_observation") is not None:

                                    file = '{}/{}/{}/{}_{}_{}.xml'.format(download_location, download_feed, year, download_feed, year, station)
                                    f = open(file, "w")
                                    f.write(res.text)
                                    f.close()
                                    print('Download successful {}'.format(data_link))

                                else:
                                    print('No data found in {}'.format(data_link))

                            else:
                                print('No xml file found for {}'.format(data_link))
                                    
                        else:
                            print('No dataset link found for {}'.format(station_feed_link))
            else:
                print('No download feed for {}'.format(url))


"""
    response = requests.get(url, headers=Headers)

    soup = BeautifulSoup(response.text, features="html.parser")
    #print(soup)
    #members = soup.find_all(rdid='')
    #members = soup.find_all('om:OM_Observation')
    members = soup.find_all('gml:featuremember')
    for member in members:
            component = member.find('om:observedproperty')
            if component is not None:
                component_links = component['xlink:href']
                data = member.find('swe:values')"""


def download_metadata(n_max_tries, max_time_per_dl, version):

    base_url = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/atom.en.xml' # base link to metadata: contains all available metadata links we might need, choose the ones we need
    url_stations = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/D_GB_all_Station.xml' # link to Description of the configuration UK air quality monitoring stations 
    url_sap = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/D_GB_SamplingPoint.xml'
    url_assess = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/C_GB_AssessmentRegime_retro.xml'
    url_sample = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/D_GB_Sample.xml'
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}
    today = date.today()

    os.makedirs('/esarchive/obs/ghost/UK_AIR/metadata/{}/network_provided/'.format(version), exist_ok=True)
    download_location = '/esarchive/obs/ghost/UK_AIR/metadata/{}/network_provided/'.format(version)


    # get sampling point meta data and all other -------------------------------------------------------------------------------------------------
    urls = [url_sap, url_stations, url_assess, url_sample]
    soups = []

    for url in urls:
        n_tries = 0
        errcode = 999
        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(url, timeout=max_time_per_dl, headers=Headers)

            if r.status_code == 200:
                soups.append(BeautifulSoup(r.text, 'xml'))
                errcode = r.status_code

            elif r.status_code == 404:
                print("No data found, error 404")
                errcode = 200

            elif r.status_code == 403:
                print("Permission denied for {}".format(url))
                errcode = 200

            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                max_time_per_dl = max_time_per_dl*2
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading metadata UK AIR')

    # create csv for all metadata
    cols = ['Sampling Point ID', 'Version ID', 'Name', 'Latitude', 'Longitude', 'Start Date', 'End Date', 'Pollutant Eionet code', 'Pollutant Link Eionet', 'Station Code', 'Station Link UK', 'Station Classification Eionet', 'Station Classification Link']
    rows = []

    # add sampling point meta data ----------------------------------------------------------------------------------------------
    stations = soups[0].find_all("aqd:AQD_SamplingPoint") 
    for station in stations:
        sampling_point_ID = station.get('gml:id')[-5:]
        version_ID = station.find('base:versionId').getText()
        name = station.find("ef:name").getText()
        lat = station.find('gml:pos').getText().split(' ')[0]
        lon = station.find('gml:pos').getText().split(' ')[1]
        start = station.find('gml:beginPosition').getText()
        stop = station.find('gml:endPosition').getText()
        pollutant = station.find('ef:observedProperty').get('xlink:href')
        pollutant_ID = pollutant.split('/')[-1]
        station_link = station.find('ef:broader').get('xlink:href')
        station_ID = station_link.split('_')[-1]
        station_class_link = station.find('aqd:stationClassification').get('xlink:href')
        station_class = station_class_link.split('/')[-1]


        rows.append({"Sampling Point ID": sampling_point_ID, 
                    "Version ID": version_ID, 
                    "Name": name, 
                    "Latitude": lat,
                    "Longitude": lon,
                    "Start Date": start,
                    "End Date": stop,
                    "Pollutant Eionet code": pollutant_ID,
                    "Pollutant Link Eionet": pollutant,
                    "Station Code": station_ID,
                    "Station Link UK": station_link,
                    "Station Classification Eionet": station_class,
                    "Station Classification Link": station_class_link}) 
  
    df = pd.DataFrame(rows, columns=cols) 

    # add station metadata -----------------------------------------------------------------------------------
    df['Site name'] = np.nan
    df['Altitude'] = np.nan
    df['Environment Type Eionet'] = np.nan
    df['Environment Type Eionet Link'] = np.nan
    df['Site Info Link UK'] = np.nan
    
    stations = soups[1].find_all("aqd:AQD_Station")

    for station in stations:
        station_code = station.find('aqd:natlStationCode').getText()
        EU_station_code = station.find('aqd:EUStationCode').getText()
        name = station.find("ef:name").getText()
        alt = station.find('aqd:altitude').getText()
        area_link = station.find('aqd:areaClassification').get('xlink:href')
        area = area_link.split('/')[-1]
        station_info = station.find('aqd:stationInfo').getText()

        for station_code_sp in set(df["Station Code"]): # loop over stations in sampling point metadata
            if (station_code_sp == station_code) or (station_code_sp == EU_station_code): # match sampling point with corresponding station
                df.loc[df['Station Code'] == station_code_sp, 'Site name'] = [str(name)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Altitude'] = [alt] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Environment Type Eionet'] = [str(area)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Environment Type Eionet Link'] = [str(area_link)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Site Info Link UK'] = [str(station_info)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])



    # add SAMPLE metadata -----------------------------------------------------------------------------------
    df['Inlet Height (m)'] = np.nan
    df['Building Distance (m)'] = np.nan
    df['Kerb Distance (m)'] = np.nan
    
    stations = soups[3].find_all("aqd:AQD_Sample")

    for station in stations:
        try:
            name_samp = station.find('gml:name').getText()
        except:
            continue
        
        try:
            inletheight = station.find('aqd:inletHeight').getText()
        except:
            inletheight = None

        try:
            builddis = station.find('aqd:buildingDistance').getText()
        except:
            builddis = None
            
        try:
            kerbdis = station.find('aqd:kerbDistance').getText()
        except:
            kerbdis = None


        for name in df["Name"]: # loop over sampling points
            if name == name_samp: # match sampling point with corresponding station
                df.loc[df['Name'] == name, 'Inlet Height (m)'] = inletheight
                df.loc[df['Name'] == name, 'Building Distance (m)'] = builddis
                df.loc[df['Name'] == name, 'Kerb Distance (m)'] = kerbdis


    # add assessment regime metadata -----------------------------------------------------------------------------------
    df['Metric'] = np.nan
    df['Assessment Type'] = np.nan
    
    stations = soups[2].find_all("aqd:AQD_AssessmentRegime")

    for station in stations:
        try:
            sampling_point_ID = station.find('aqd:samplingPointAssessmentMetadata').get('xlink:href').split('_')[-1]
        except:
            continue
        
        try:
            metric = station.find('aqd:reportingMetric').get('xlink:href').split('/')[-1]
        except:
            metric = None

        try:
            assessment_type = station.find('aqd:assessmentType').get('xlink:href').split('/')[-1]
        except:
            assessment_type = None


        for samp_ID in df["Sampling Point ID"]: # loop over sampling points
            if samp_ID == sampling_point_ID: # match sampling point with corresponding station
                df.loc[df['Sampling Point ID'] == samp_ID, 'Metric'] = str(metric) 
                df.loc[df['Sampling Point ID'] == samp_ID, 'Assessment Type'] = str(assessment_type) 


    # write csv 
    df.drop_duplicates().to_csv(download_location+'UK_AIR_META_all_{}.csv'.format(today.strftime('%Y%m%d')), index=False)
    
    # create json in desired shape from current metadata file
    json_metadata_now = {}  
    with open(download_location+'UK_AIR_META_all_{}.csv'.format(today.strftime('%Y%m%d')), encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['Sampling Point ID']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata_now[key] = row

    """
    # create json from original metadata file
    json_metadata = {}
    with open('/esarchive/obs/ghost/UK_AIR/metadata/{}/network_provided/U.format(version)K_AIR_META.csv', 'r', encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['station_code']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata[key] = row

    with open('/esarchive/obs/ghost/UK_AIR/metadata/{}/processed/UK_AIR_META.json'.format(version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4))"""
        
    
    # read standardised file to compare!
    with open('/esarchive/obs/ghost/UK_AIR/metadata/{}/processed/UK_AIR_META.json'.format(version), 'r', encoding='utf-8') as f:
        json_metadata = json.loads(f.read())

        for station in json_metadata: # loop through all the old stations
            if station in json_metadata_now.keys(): # if station is in current meta data, go on
                for parameter in json_metadata[station]:
                    if parameter in json_metadata_now[station].keys(): # check if column of csv exists in new file
                        if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                            # if different value, append the standardised metadeta file
                            print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                            json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                            json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                        else:
                            pass
                    else:
                        print('{} not in new metadata file'.format(parameter))
            else:
                print('Station {} was abolished'.format(station))

        for station in json_metadata_now: # loop through all the new stations
            for parameter in json_metadata_now[station]: # loop through all the parameters
                if station in json_metadata.keys(): # if station is in old meta data
                    pass # comparison was done before
                else: # new station appeared!
                    print('New station {}'.format(station))
                    json_metadata.update({station: json_metadata_now[station]})
                # is there a new parameter that wasn't in the old file?
                if parameter in json_metadata[station].keys():
                    pass # parameter (column) is already there
                else:
                    print('{} is new'.format(parameter))
                    json_metadata[station].update({parameter: json_metadata_now[station][parameter]})

    # safe
    with open('/esarchive/obs/ghost/UK_AIR/metadata/{}/processed/UK_AIR_META.json'.format(version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata_now, indent=4))
            

    # get station metadata ----------------------------------------------------------------------------------------------------------------------
    # n_tries = 0
    # errcode = 999
    # while (n_tries < n_max_tries) and (errcode != 200):
    #     r = requests.get(url, timeout=max_time_per_dl, headers=Headers)

    #     if r.status_code == 200:
    #         soup = BeautifulSoup(r.text, 'xml')
    #         errcode = r.status_code

    #     elif r.status_code == 404:
    #         print("No data found, error 404")
    #         errcode = 200

    #     elif r.status_code == 403:
    #         print("Permission denied for {}".format(url))
    #         errcode = 200

    #     else:
    #         # try again
    #         print('Response error {}, attempt {}'.format(r.status_code, n_tries))
    #         errcode = r.status_code
    #         n_tries += 1
    #         max_time_per_dl = max_time_per_dl*2
    #         time.sleep(n_tries ** 2) # wait a lil more every time 

    # if n_tries == n_max_tries:
    #     print('Failed downloading metadata UK AIR')


    # cols = ['UK-AIR ID', 'EU Site ID', 'Version ID', 'Site Name', 'Latitude', 'Longitude', 'Altitude', 'Start Date', 'End Date', 'Environment Type', 'Station Info']
    # rows = []

    # stations = soup.find_all("aqd:AQD_Station")
    # for station in stations:
    #     station_code = station.find('aqd:natlStationCode').getText()
    #     EU_station_code = station.find('aqd:EUStationCode').getText()
    #     version_ID = station.find('base:versionId').getText()
    #     name = station.find("ef:name").getText()
    #     lat = station.find('gml:pos').getText().split(' ')[0]
    #     lon = station.find('gml:pos').getText().split(' ')[1]
    #     alt = station.find('aqd:altitude').getText()
    #     start = station.find('gml:beginPosition').getText()
    #     stop = station.find('gml:endPosition').getText()
    #     area = station.find('aqd:areaClassification').get('xlink:href')
    #     station_info = station.find('aqd:stationInfo').getText()

    #     rows.append({"UK-AIR ID": station_code, 
    #                 "EU Site ID": EU_station_code, 
    #                 "Version ID": version_ID, 
    #                 "Site Name": name, 
    #                 "Latitude": lat,
    #                 "Longitude": lon,
    #                 "Altitude (m)": alt,
    #                 "Start Date": start,
    #                 "End Date": stop,
    #                 "Environment Type": area,
    #                 "Station Info": station_info}) 
  
    # df = pd.DataFrame(rows, columns=cols) 
    # df.drop_duplicates().to_csv(download_location+'UK_AIR_META_station_{}.csv'.format(today.strftime('%Y%m%d')), index=False) 

    # # create json in desired shape from current metadata file
    # json_metadata_now = {}  
    # with open(download_location+'UK_AIR_META_station_{}.csv'.format(today.strftime('%Y%m%d')), encoding='utf-8') as file:
    #     csv_filedata = csv.DictReader(file)

    #     for row in csv_filedata:  
    #         key = row['UK-AIR ID']
    #         update_date = today.strftime('%Y-%m-%d')
    #         for parameter in row:
    #             row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
    #         json_metadata_now[key] = row

    # # read standardised file to compare!
    # with open('/esarchive/obs/ghost/UK_AIR/metadata/{}/processed/UK_AIR_META_station.json', 'r', encoding='utf-8') as f:
    #     json_metadata = json.loads(f.read())

    #     for station in json_metadata: # loop through all the old stations
    #         if station in json_metadata_now.keys(): # if station is in current meta data, go on
    #             for parameter in json_metadata[station]:
    #                 if parameter in json_metadata_now[station].keys(): # check if column of csv exists in new file
    #                     if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
    #                         # if different value, append the standardised metadeta file
    #                         print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
    #                         json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
    #                         json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
    #                     else:
    #                         pass
    #                 else:
    #                     print('{} not in new metadata file'.format(parameter))
    #         else:
    #             print('Station {} was abolished'.format(station))

    #     for station in json_metadata_now: # loop through all the new stations
    #         for parameter in json_metadata_now[station]: # loop through all the parameters
    #             if station in json_metadata.keys(): # if station is in old meta data
    #                 pass # comparison was done before
    #             else: # new station appeared!
    #                 print('New station {}'.format(station))
    #                 json_metadata.update({station: json_metadata_now[station]})
    #             # is there a new parameter that wasn't in the old file?
    #             if parameter in json_metadata[station].keys():
    #                 pass # parameter (column) is already there
    #             else:
    #                 print('{} is new'.format(parameter))
    #                 json_metadata[station].update({parameter: json_metadata_now[station][parameter]})
    
    # # safe
    # with open('/esarchive/obs/ghost/UK_AIR/metadata/{}/processed/UK_AIR_META_station.json', 'w', encoding='utf-8') as f:
    #     f.write(json.dumps(json_metadata, indent=4))