import requests
import time
import pandas as pd
from datetime import date
import os.path
import urllib
import shutil
import zipfile
import re
import glob
import csv
import json
import metadata_parsing_functions

from selenium import webdriver
from bs4 import BeautifulSoup
from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC



def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    baseurl = 'https://www.miteco.gob.es/es/calidad-y-evaluacion-ambiental/temas/atmosfera-y-calidad-del-aire/calidad-del-aire/evaluacion-datos/datos/datos-2001-2021.html'

    if mode == 'all':
        bdate = date(2001, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today()
        os.makedirs('{}/MITECO/original_files/{}/'.format(dl_root,version), exist_ok=True)
        download_location = '{}/MITECO/original_files/{}/'.format(dl_root,version)

    elif mode == 'nrt':
        print("nrt not available MITECO")

    else:
        print('time mode inapplicable')


    # create date array; format to YYYYMMDD
    years_until_2015 = pd.date_range(bdate, date(2015, 1, 1), freq='Y').strftime('%Y').tolist()
    years_after_2015 = pd.date_range(date(2016, 1, 1), edate, freq='Y').strftime('%Y').tolist()
    print(years_after_2015)

    # set up driver
    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")

    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    # open url
    driver.get(baseurl)

    # find zip links
    html = driver.page_source
    soup = BeautifulSoup(html, features="html.parser")
    zip_links = soup.find_all("a", href=re.compile(r".zip"))
    
    for zip_link in zip_links:
        filename = zip_link.get("href").rpartition('/')[-1]
        url = 'https://www.miteco.gob.es/{}'.format(zip_link.get("href"))
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(url, timeout=max_time_per_dl)
            if r.status_code == 200:
                urllib.request.urlretrieve(url, download_location+filename)
                print('Downloaded {}'.format(filename))
                # unzip
                with zipfile.ZipFile(download_location+filename, 'r') as zip_ref:
                    zip_ref.extractall(download_location)
                os.remove(download_location+filename)

                errcode = r.status_code

            elif r.status_code == 404:
                print("No data found, error 404")
                errcode = 200

            elif r.status_code == 403:
                print("Permission denied for {}".format(url))
                errcode = 200

            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                max_time_per_dl = max_time_per_dl*2
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
        time.sleep(1)

    # go to hyperlinks

    for year in years_after_2015:

        driver.get('https://www.miteco.gob.es/es/calidad-y-evaluacion-ambiental/temas/atmosfera-y-calidad-del-aire/calidad-del-aire/evaluacion-datos/datos/datos_oficiales_{}.html'.format(year))
        if year == '2022':
                driver.get('https://www.miteco.gob.es/es/calidad-y-evaluacion-ambiental/temas/atmosfera-y-calidad-del-aire/calidad-del-aire/evaluacion-datos/datos/datos-oficiales-2022.html')

        time.sleep(3)

        html = driver.page_source
        soup = BeautifulSoup(html, features="html.parser")
        zip_links = soup.find_all("a", href=re.compile(r".zip"))
        
        
        for zip_link in zip_links:
            filename = zip_link.get("href").rpartition('/')[-1]
            #print(filename)
            url = 'https://www.miteco.gob.es/{}'.format(zip_link.get("href"))
            
            if year == '2022':
                driver.get(url)
                time.sleep(5)
                # unzip
                for zip_file in glob.glob(download_location+'*.zip'):
                    with zipfile.ZipFile(zip_file, 'r') as zip_ref:
                        zip_ref.extractall(download_location)
                    os.remove(zip_file)

                continue
            
            n_tries = 0
            errcode = 999

            while (n_tries < n_max_tries) and (errcode != 200):
                r = requests.get(url, timeout=max_time_per_dl)
                if r.status_code == 200:
                    urllib.request.urlretrieve(url, download_location+filename)
                    print('Downloaded {}'.format(filename))

                    # unzip
                    with zipfile.ZipFile(download_location+filename, 'r') as zip_ref:
                        zip_ref.extractall(download_location)
            
                    os.remove(download_location+filename)
                    errcode = r.status_code

                elif r.status_code == 404:
                    print("No data found, error 404")
                    errcode = 200

                elif r.status_code == 403:
                    print("Permission denied for {}".format(url))
                    errcode = 200

                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    max_time_per_dl = max_time_per_dl*2
                    time.sleep(n_tries ** 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
            time.sleep(1)

    # delete metadata
    for metadata in glob.glob(download_location+'*.xls'):
        os.remove(metadata)

    # move files around
    alldirectories =[directory for directory in os.listdir(download_location) if not os.path.isfile(os.path.join(download_location, directory))]
    for directory in alldirectories:
        allfiles = os.listdir(os.path.join(download_location, directory))
        for f in allfiles:
            os.rename(os.path.join(download_location, directory, f), os.path.join(download_location, f))
        try:
            shutil.rmtree(os.path.join(download_location, directory))
        except:
            pass

    driver.close()


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    last_year_meta_av = 2022
    url_metadata = 'https://www.miteco.gob.es/content/dam/miteco/es/calidad-y-evaluacion-ambiental/sgalsi/atm%C3%B3sfera-y-calidad-del-aire/evaluaci%C3%B3n-{}/Metainformacion{}.xlsx'.format(last_year_meta_av, last_year_meta_av)
    #url_metadata = 'https://www.miteco.gob.es/es/calidad-y-evaluacion-ambiental/temas/atmosfera-y-calidad-del-aire/calidad-del-aire/evaluacion-datos/datos/datos-oficiales-2022.html'
    download_location = dl_root+"/MITECO/metadata/"+version+"/network_provided/MITECO_META_{}.xlsx"

    today = date.today()

    r = requests.get(url_metadata, timeout=max_time_per_dl)
    n_tries = 0
    errcode = 999

    
    while (n_tries < n_max_tries) and (errcode != 200):
        if r.status_code == 200:
            urllib.request.urlretrieve(url_metadata, download_location.format(today.strftime('%Y%m%d')))
            #with open(download_location.format(today.strftime('%Y%m%d')), 'wb') as outfile:
            #    outfile.write(r.content)
            print('Downloaded metadata')
            errcode = r.status_code
        elif r.status_code == 404:
            print("No metadata found, error 404")
            errcode = 200
        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_metadata, n_tries, max_time_per_dl, errcode))
    time.sleep(1)

    # convert to csv
    metadata = pd.read_excel('{}/MITECO/metadata/{}/network_provided/MITECO_META_{}.xlsx'.format(dl_root, version, today.strftime('%Y%m%d')), engine='openpyxl', sheet_name='Estaciones').fillna('')
    metadata.to_csv('{}/MITECO/metadata/{}/network_provided/MITECO_META_{}.csv'.format(dl_root, version,today.strftime('%Y%m%d')), index=False, header=True, sep=';')

    os.remove(download_location.format(today.strftime('%Y%m%d')))
    #shutil.copyfile('{}/MITECO/metadata/{}/network_provided/MITECO_META_{}.csv'.format(dl_root, version,today.strftime('%Y%m%d')), '{}/MITECO/metadata/{}/network_provided/MITECO_META_{}.csv'.format(dl_root, version,last_year_meta_av))
    
    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)