from bs4 import BeautifulSoup
import requests
import time
from datetime import date
from datetime import timedelta
import os.path
import pandas as pd
import numpy as np
import csv
import json
import metadata_parsing_functions

def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    base_url = 'https://uk-air.defra.gov.uk/data/atom-dls/{}/{}/atom.en.xml'
    #base_url_til_1996 = 'https://uk-air.defra.gov.uk/data/atom-dls/observations/auto/GB_FixedObservations_{}_CRD.xml'
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    if mode == 'all':
        bdate = date(1970, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)

        os.makedirs('{}/UK_AIR/original_files/{}/'.format(dl_root,version), exist_ok=True)
        download_location = '{}/UK_AIR/original_files/{}/'.format(dl_root,version)

    elif mode == 'nrt':
        version = mode
        bdate = start 
        edate = stop + timedelta(days = 365)

        os.makedirs('{}/UK_AIR/original_files/nrt/'.format(dl_root), exist_ok=True)
        download_location = '{}/UK_AIR/original_files/nrt/'.format(dl_root)

    else:
        print('time mode inapplicable')


    # create date array, daily; format YYYYMMDD
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    print(years)
    download_feeds = ['auto', 'non-auto', 'aggregated']
   

    for download_feed in download_feeds:

        for year in years:

            os.makedirs('{}/UK_AIR/original_files/{}/{}/{}/'.format(dl_root, version, download_feed, year), exist_ok=True)

            # get feed for each year and mode
            url = base_url.format(download_feed, year)
            r = requests.get(url, timeout=max_time_per_dl, headers=Headers)
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, features="html.parser")

                # find link to station feed
                station_feeds = soup.find_all(title="Feed containing the dataset in several formats")
                for station_feed in station_feeds:

                    if station_feed is not None:
                        station_feed_link = station_feed['href']

                        # request new page url
                        response = requests.get(station_feed_link, timeout=max_time_per_dl, headers=Headers)
                        if response.status_code == 200:
                            soup_dataset = BeautifulSoup(response.text, features="html.parser")
                            data_link = soup_dataset.find("link", type='application/gml+xml;charset=utf-8')['href']
                            print(data_link)
                            station = data_link.split('_')[-1]
                            station = station.split('.')[0]

                            # request download
                            res = requests.get(data_link, timeout=max_time_per_dl, headers=Headers)
                            if res.status_code == 200:

                                # check if xml contains any data
                                soup_data = BeautifulSoup(res.text, features="html.parser")
                                if soup_data.find("om:om_observation") is not None:

                                    file = '{}/{}/{}/{}_{}_{}.xml'.format(download_location, download_feed, year, download_feed, year, station)
                                    f = open(file, "w")
                                    f.write(res.text)
                                    f.close()
                                    print('Download successful {}'.format(data_link))

                                else:
                                    print('No data found in {}'.format(data_link))

                            else:
                                print('No xml file found for {}'.format(data_link))
                                    
                        else:
                            print('No dataset link found for {}'.format(station_feed_link))
            else:
                print('No download feed for {}'.format(url))


"""
    response = requests.get(url, headers=Headers)

    soup = BeautifulSoup(response.text, features="html.parser")
    #print(soup)
    #members = soup.find_all(rdid='')
    #members = soup.find_all('om:OM_Observation')
    members = soup.find_all('gml:featuremember')
    for member in members:
            component = member.find('om:observedproperty')
            if component is not None:
                component_links = component['xlink:href']
                data = member.find('swe:values')"""


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    base_url = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/atom.en.xml' # base link to metadata: contains all available metadata links we might need, choose the ones we need
    url_stations = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/D_GB_all_Station.xml' # link to Description of the configuration UK air quality monitoring stations 
    url_sap = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/D_GB_SamplingPoint.xml'
    url_assess = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/C_GB_AssessmentRegime_retro.xml'
    url_sample = 'https://uk-air.defra.gov.uk/data/atom-dls/metadata/D_GB_Sample.xml'
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}
    today = date.today()

    os.makedirs('{}/UK_AIR/metadata/{}/network_provided/'.format(dl_root, version), exist_ok=True)
    download_location = '{}/UK_AIR/metadata/{}/network_provided/'.format(dl_root, version)


    # get sampling point meta data and all other -------------------------------------------------------------------------------------------------
    urls = [url_sap, url_stations, url_assess, url_sample]
    soups = []

    for url in urls:
        n_tries = 0
        errcode = 999
        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(url, timeout=max_time_per_dl, headers=Headers)

            if r.status_code == 200:
                soups.append(BeautifulSoup(r.text, 'xml'))
                errcode = r.status_code

            elif r.status_code == 404:
                print("No data found, error 404")
                errcode = 200

            elif r.status_code == 403:
                print("Permission denied for {}".format(url))
                errcode = 200

            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                max_time_per_dl = max_time_per_dl*2
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading metadata UK AIR')

    # create csv for all metadata
    cols = ['Sampling Point ID', 'Version ID', 'Name', 'Latitude', 'Longitude', 'Start Date', 'End Date', 'Pollutant Eionet code', 'Pollutant Link Eionet', 'Station Code', 'Station Link UK', 'Station Classification Eionet', 'Station Classification Link']
    rows = []

    # add sampling point meta data ----------------------------------------------------------------------------------------------
    stations = soups[0].find_all("aqd:AQD_SamplingPoint") 
    for station in stations:
        sampling_point_ID = station.get('gml:id')[-5:]
        version_ID = station.find('base:versionId').getText()
        name = station.find("ef:name").getText()
        lat = station.find('gml:pos').getText().split(' ')[0]
        lon = station.find('gml:pos').getText().split(' ')[1]
        start = station.find('gml:beginPosition').getText()
        stop = station.find('gml:endPosition').getText()
        pollutant = station.find('ef:observedProperty').get('xlink:href')
        pollutant_ID = pollutant.split('/')[-1]
        station_link = station.find('ef:broader').get('xlink:href')
        station_ID = station_link.split('_')[-1]
        station_class_link = station.find('aqd:stationClassification').get('xlink:href')
        station_class = station_class_link.split('/')[-1]


        rows.append({"Sampling Point ID": sampling_point_ID, 
                    "Version ID": version_ID, 
                    "Name": name, 
                    "Latitude": lat,
                    "Longitude": lon,
                    "Start Date": start,
                    "End Date": stop,
                    "Pollutant Eionet code": pollutant_ID,
                    "Pollutant Link Eionet": pollutant,
                    "Station Code": station_ID,
                    "Station Link UK": station_link,
                    "Station Classification Eionet": station_class,
                    "Station Classification Link": station_class_link}) 
  
    df = pd.DataFrame(rows, columns=cols) 

    # add station metadata -----------------------------------------------------------------------------------
    df['Site name'] = np.nan
    df['Altitude'] = np.nan
    df['Environment Type Eionet'] = np.nan
    df['Environment Type Eionet Link'] = np.nan
    df['Site Info Link UK'] = np.nan
    
    stations = soups[1].find_all("aqd:AQD_Station")

    for station in stations:
        station_code = station.find('aqd:natlStationCode').getText()
        EU_station_code = station.find('aqd:EUStationCode').getText()
        name = station.find("ef:name").getText()
        alt = station.find('aqd:altitude').getText()
        area_link = station.find('aqd:areaClassification').get('xlink:href')
        area = area_link.split('/')[-1]
        station_info = station.find('aqd:stationInfo').getText()

        for station_code_sp in set(df["Station Code"]): # loop over stations in sampling point metadata
            if (station_code_sp == station_code) or (station_code_sp == EU_station_code): # match sampling point with corresponding station
                df.loc[df['Station Code'] == station_code_sp, 'Site name'] = [str(name)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Altitude'] = [alt] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Environment Type Eionet'] = [str(area)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Environment Type Eionet Link'] = [str(area_link)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])
                df.loc[df['Station Code'] == station_code_sp, 'Site Info Link UK'] = [str(station_info)] * len(df.loc[df['Station Code'] == station_code_sp]['Site name'])



    # add SAMPLE metadata -----------------------------------------------------------------------------------
    df['Inlet Height (m)'] = np.nan
    df['Building Distance (m)'] = np.nan
    df['Kerb Distance (m)'] = np.nan
    
    stations = soups[3].find_all("aqd:AQD_Sample")

    for station in stations:
        try:
            name_samp = station.find('gml:name').getText()
        except:
            continue
        
        try:
            inletheight = station.find('aqd:inletHeight').getText()
        except:
            inletheight = None

        try:
            builddis = station.find('aqd:buildingDistance').getText()
        except:
            builddis = None
            
        try:
            kerbdis = station.find('aqd:kerbDistance').getText()
        except:
            kerbdis = None


        for name in df["Name"]: # loop over sampling points
            if name == name_samp: # match sampling point with corresponding station
                df.loc[df['Name'] == name, 'Inlet Height (m)'] = inletheight
                df.loc[df['Name'] == name, 'Building Distance (m)'] = builddis
                df.loc[df['Name'] == name, 'Kerb Distance (m)'] = kerbdis


    # add assessment regime metadata -----------------------------------------------------------------------------------
    df['Metric'] = np.nan
    df['Assessment Type'] = np.nan
    
    stations = soups[2].find_all("aqd:AQD_AssessmentRegime")

    for station in stations:
        try:
            sampling_point_ID = station.find('aqd:samplingPointAssessmentMetadata').get('xlink:href').split('_')[-1]
        except:
            continue
        
        try:
            metric = station.find('aqd:reportingMetric').get('xlink:href').split('/')[-1]
        except:
            metric = None

        try:
            assessment_type = station.find('aqd:assessmentType').get('xlink:href').split('/')[-1]
        except:
            assessment_type = None


        for samp_ID in df["Sampling Point ID"]: # loop over sampling points
            if samp_ID == sampling_point_ID: # match sampling point with corresponding station
                df.loc[df['Sampling Point ID'] == samp_ID, 'Metric'] = str(metric) 
                df.loc[df['Sampling Point ID'] == samp_ID, 'Assessment Type'] = str(assessment_type) 


    # write csv 
    df.drop_duplicates().to_csv(download_location+'UK_AIR_META_{}.csv'.format(today.strftime('%Y%m%d')), index=False)

    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)