from selenium import webdriver
from datetime import date
import os.path
import time
import csv
import json

from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException



def download_data(mode, version, n_max_tries, max_time_per_dl):

    baseurl = 'https://nadp.slh.wisc.edu/networks/atmospheric-mercury-network/'

    if mode == 'all':
        os.makedirs('/esarchive/obs/ghost/US_NADP_AMNet/original_files/{}/temp/'.format(version), exist_ok=True)
        download_location = '/esarchive/obs/ghost/US_NADP_AMNet/original_files/{}/temp/'.format(version)

    elif mode == 'nrt':
        download_location = '/esarchive/obs/ghost/US_NADP_AMNet/original_files/nrt/temp/'

    else:
        print('time mode inapplicable')


    # set up driver
    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")

    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)
    n_tries = 0
    errcode = 999

    while (n_tries < n_max_tries) and (errcode != 200):
        
        try:
            # open url
            driver.get(baseurl)
            WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.ID, 'sites-list'))) # wait till loaded

            # select time resolution
            dropdown_element = driver.find_element(By.ID, 'sites-list')
            select = Select(dropdown_element)
            options = [opt.get_attribute("text") for opt in select.options]
            #print(options)

            select.select_by_visible_text("All Sites")

            #WebDriverWait(driver, max_time_per_dl).until(EC.element_to_be_clickable((By.ID, 'invalid'))) # wait till loaded
            time.sleep(max_time_per_dl)
            invalid_box = driver.find_element(By.ID, 'invalid')
            invalid_box.click()

            # download
            #WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.ID, 'generate-button-text'))) # wait till loaded
            time.sleep(max_time_per_dl)
            driver.find_element(By.ID, 'generate-button-text').click()

            # wait until download finished
            while not os.path.exists("{}AMNET-ALL-h.csv".format(download_location)):
                time.sleep(1)

                if os.path.isfile("{}AMNET-ALL-h.csv".format(download_location)):
                    print('AMNET-ALL-h.csv download successful')
                    errcode = 200

        except TimeoutException as e:
            print(e)
            max_time_per_dl = max_time_per_dl*2 # set waiting time to double
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue

        except WebDriverException as e:
            print(e) 
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue
        
        except Exception as e:
            print(e)
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue
        
    driver.close()

    if n_tries == n_max_tries:
        print('Failed downloading US_NADP_AMNet data {} times in {} seconds'.format(n_tries, max_time_per_dl))

    os.rename("{}AMNET-ALL-h.csv".format(download_location), "/esarchive/obs/ghost/US_NADP_AMNet/original_files/{}/AMNET-ALL-h.csv".format(version))




def download_metadata(n_max_tries, max_time_per_dl):

    baseurl = 'https://nadp.slh.wisc.edu/networks/atmospheric-mercury-network/'
    #os.makedirs('/esarchive/obs/ghost/US_NADP_AMNet/metadata/network_provided/', exist_ok=True)
    download_location = '/esarchive/obs/ghost/US_NADP_AMNet/metadata/network_provided/'
    today = date.today()

    # set up driver
    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")

    n_tries = 0
    errcode = 999
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)
    driver.maximize_window()

    while (n_tries < n_max_tries) and (errcode != 200):
        try:
            # open url
            driver.get(baseurl)

            #WebDriverWait(driver, max_time_per_dl).until(EC.element_to_be_clickable((By.ID, 'invalid'))) # wait till loaded
            time.sleep(max_time_per_dl)
            invalid_box = driver.find_element(By.ID, 'download-show-inactive')
            driver.execute_script("arguments[0].click()", invalid_box)
            

            # download
            #WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.ID, 'generate-button-text'))) # wait till loaded
            time.sleep(max_time_per_dl)
            bttn = driver.find_element(By.ID, 'network-data-submit')
            driver.execute_script("arguments[0].click()", bttn)


            # wait until download finished
            while not os.path.exists(download_location+'amnet.csv'):
                time.sleep(1)

                if os.path.isfile(download_location+'amnet.csv'):
                    print('Amnet metadata download successful')
                    errcode = 200
                    continue

        except TimeoutException as e:
            print(e)
            max_time_per_dl = max_time_per_dl*2 # set waiting time to double
            n_tries = n_tries+1
            continue
        
        except WebDriverException as e:
            print(e) 
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue
        
        except:
            print("Unknown error")
            max_time_per_dl = max_time_per_dl*2 # set waiting time to double
            n_tries = n_tries+1
            continue


    if n_tries == n_max_tries:
        print('Failed downloading AMNet metadata {} times in {} seconds'.format(n_tries, max_time_per_dl))

    driver.close()

    os.rename(download_location+'amnet.csv', download_location+'US_NADP_AMNet_META_{}.csv'.format(today.strftime('%Y%m%d')))

    # create json from original metadata file =====================================================================================
    """json_metadata = {}
    with open('/esarchive/obs/ghost/US_NADP_AMNet/metadata/network_provided/US_NADP_AMNet_META.csv', 'r') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['siteId']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata[key] = row

    with open('/esarchive/obs/ghost/US_NADP_AMNet/metadata/processed/US_NADP_AMNet_META.json', 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4))

    """
    # create json in desired shape from current metadata file
    json_metadata_now = {}  
    with open(download_location+'US_NADP_AMNet_META_{}.csv'.format(today.strftime('%Y%m%d'))) as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['siteId']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata_now[key] = row

    
    # read standardised file to compare!
    with open('/esarchive/obs/ghost/US_NADP_AMNet/metadata/processed/US_NADP_AMNet_META.json', 'r', encoding='utf-8') as f:
        json_metadata = json.loads(f.read())

        for station in json_metadata: # loop through all the old stations
            if station in json_metadata_now.keys(): # if station is in current meta data, go on
                for parameter in json_metadata[station]:
                    if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                        # if different value, append the standardised metadeta file
                        print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                        json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                        json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                    else:
                        pass
            else:
                print('Station {} was abolished'.format(station))

        for station in json_metadata_now: # loop through all the new stations
            if station in json_metadata.keys(): # if station is in old meta data
                pass # comparison was done before
            else: # new station appeared!
                print('New station {}'.format(station))
                json_metadata.update({station: json_metadata_now[station]})


    # safe
    with open('/esarchive/obs/ghost/US_NADP_AMNet/metadata/processed/US_NADP_AMNet_META.json', 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4))