from selenium import webdriver
from bs4 import BeautifulSoup
import requests
import hashlib
from datetime import date
from datetime import timedelta
import pandas
import os.path
import urllib
import time
import ssl 
import zipfile
from compare_two_files import compare_files

from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC




def scraper(mode, version):

    baseurl = 'https://nadp.slh.wisc.edu/networks/atmospheric-mercury-network/'

    if mode == 'all':
        bdate = date(2013, 12, 1) #date(1960, 1, 1) # date before record starts
        edate = date(2024, 1, 1) #date.today() - timedelta(days = 1)

        os.makedirs('/esarchive/obs/ghost/US_NADP_AMNet/original_files/{}/'.format(version), exist_ok=True)

        download_location = '/esarchive/obs/ghost/US_NADP_AMNet/original_files/{}/'.format(version)

    elif mode == 'nrt':
        bdate = date(2024, 1, 1) #date.today() - timedelta(days = 1)
        edate = date.today() - timedelta(days = 1)
        download_location = '/esarchive/obs/ghost/US_NADP_AMNet/original_files/nrt/'

    else:
        print('time mode inapplicable')


    # set up driver
    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")

    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    # open url
    driver.get(baseurl)
    WebDriverWait(driver, 60).until(EC.presence_of_all_elements_located((By.ID, 'sites-list'))) # wait till loaded

    # select time resolution
    dropdown_element = driver.find_element(By.ID, 'sites-list')
    select = Select(dropdown_element)
    options = [opt.get_attribute("text") for opt in select.options]
    print(options)

    select.select_by_visible_text("All Sites")
    time.sleep(3)

    invalid_box = driver.find_element(By.ID, 'invalid')
    invalid_box.click()

    time.sleep(3)

    # download
    driver.find_element(By.ID, 'generate-button-text').click()

    # wait until download finished
    while not os.path.exists("{}AMNET-ALL-h.csv".format(download_location)):
        time.sleep(1)

        if os.path.isfile("{}AMNET-ALL-h.csv".format(download_location)):
            print('AMNET-ALL-h.csv download successful')

    driver.close()