from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests

import time
from datetime import date
from datetime import timedelta
import os.path
import urllib
import pandas as pd
import re
import xml


def scraper(mode, version):

    base_url = 'https://uk-air.defra.gov.uk/data/atom-dls/{}/{}/atom.en.xml'
    #base_url_til_1996 = 'https://uk-air.defra.gov.uk/data/atom-dls/observations/auto/GB_FixedObservations_{}_CRD.xml'
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    if mode == 'all':
        bdate = date(2022, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)
        #edate = date(1996, 1, 1) #date(1960, 1, 1) # date before record starts

        os.makedirs('/esarchive/obs/ghost/UK_AIR/original_files/{}/'.format(version), exist_ok=True)

        download_location = '/esarchive/obs/ghost/UK_AIR/original_files/{}/'.format(version)

    elif mode == 'nrt':
        version = mode
        bdate = date.today() 
        edate = date.today() + timedelta(days = 365)
        download_location = '/esarchive/obs/ghost/UK_AIR/original_files/nrt/'

    else:
        print('time mode inapplicable')


    # create date array, daily; format YYYYMMDD
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    download_feeds = ['auto'] #, 'non-auto', 'aggregated']
   

    for download_feed in download_feeds:

        for year in years:

            os.makedirs('/esarchive/obs/ghost/UK_AIR/original_files/{}/{}/{}/'.format(version, download_feed, year), exist_ok=True)

            # get feed for each year and mode
            url = base_url.format(download_feed, year)
            r = requests.get(url, timeout=60, headers=Headers)
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, features="html.parser")

                # find link to station feed
                station_feeds = soup.find_all(title="Feed containing the dataset in several formats")
                for station_feed in station_feeds:

                    if station_feed is not None:
                        station_feed_link = station_feed['href']

                        # request new page url
                        response = requests.get(station_feed_link, timeout=60, headers=Headers)
                        if response.status_code == 200:
                            soup_dataset = BeautifulSoup(response.text, features="html.parser")
                            data_link = soup_dataset.find("link", type='application/gml+xml;charset=utf-8')['href']
                            station = data_link.split('_')[-1]
                            station = station.split('.')[0]

                            # request download
                            res = requests.get(data_link, timeout=60, headers=Headers)
                            if res.status_code == 200:

                                # check if xml contains any data
                                soup_data = BeautifulSoup(res.text, features="html.parser")
                                if soup_data.find("om:om_observation") is not None:

                                    file = '{}/{}/{}/{}_{}_{}.xml'.format(download_location, download_feed, year, download_feed, year, station)
                                    f = open(file, "w")
                                    f.write(res.text)
                                    f.close()
                                    print('Download successful {}'.format(data_link))

                                else:
                                    print('No data found in {}'.format(data_link))

                            else:
                                print('No xml file found for {}'.format(data_link))
                                    
                        else:
                            print('No dataset link found for {}'.format(station_feed_link))
            else:
                print('No download feed for {}'.format(url))


"""
    response = requests.get(url, headers=Headers)

    soup = BeautifulSoup(response.text, features="html.parser")
    #print(soup)
    #members = soup.find_all(rdid='')
    #members = soup.find_all('om:OM_Observation')
    members = soup.find_all('gml:featuremember')
    for member in members:
            component = member.find('om:observedproperty')
            if component is not None:
                component_links = component['xlink:href']
                data = member.find('swe:values')"""
            