from selenium import webdriver
from bs4 import BeautifulSoup
import requests
import hashlib
from datetime import date
from datetime import timedelta
import pandas
import os.path
import urllib
import time
import ssl 
import zipfile
from compare_two_files import compare_files



def scraper(mode, version):

    ssl._create_default_https_context = ssl._create_unverified_context
    
    variables_per_year = ["ozone", "hourly_gas"] 
    variables_single_file = ["drychem", "vischem"]

    base_url = "https://gaftp.epa.gov/castnet/CASTNET_Outgoing/data/"


    os.makedirs('/esarchive/obs/ghost/US_EPA_CASTNET/original_files/{}/'.format(version), exist_ok=True)

    if mode == 'all':
        bdate = date(1980, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)
        download_location = '/esarchive/obs/ghost/US_EPA_CASTNET/original_files/'+version+'/{}/{}_{}.zip'
        storage_location = '/esarchive/obs/ghost/US_EPA_CASTNET/original_files/'+version+'/{}/{}_{}.zip'

    elif mode == 'nrt':
        bdate = date.today() 
        edate = date.today() + timedelta(days = 365)
        download_location = '/esarchive/obs/ghost/US_EPA_CASTNET/original_files/nrt/{}/temp/{}_{}.zip'
        storage_location = '/esarchive/obs/ghost/US_EPA_CASTNET/original_files/nrt/{}/{}_{}.zip'


    else:
        print('time mode inapplicable')

    # create date array, per year
    years = pandas.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()

    print(years)

    # ozone and hourly gas
    for variable in variables_per_year:

        for year in years:

            url = '{}{}_{}.zip'.format(base_url, variable, year)
            r = requests.get(url, timeout=120, verify=False)

            if r.status_code == 200:
                urllib.request.urlretrieve(url, download_location.format(variable, variable, year))
                print('Downloaded {}'.format(url))
            else:
                print('No {}'.format(url))

            time.sleep(1)



    # drychem and vichem
    for variable in variables_single_file:

        url = '{}{}.zip'.format(base_url, variable)
        r = requests.get(url, timeout=120, verify=False)

        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location.format(variable, variable, "allyears"))
            print('Downloaded {}'.format(url))
        else:
            print('No {}'.format(url))

        time.sleep(1)


    # compare old and new file
    if mode == "nrt":
        for variable in variables_per_year:

            # compare old and new files after download
            same_files = compare_files(download_location.format(variable, variable, year), storage_location.format(variable, variable, year))

            if same_files == True:
                # delete file in temp
                os.remove(download_location.format(variable, variable, year))
            else:
                # delete old file, move new file
                os.remove(storage_location.format(variable, variable, year))
                os.rename(download_location.format(variable, variable, year), storage_location.format(variable, variable, year))

                # unzip
                with zipfile.ZipFile(storage_location.format(variable, variable, year), 'r') as zip_ref:
                    zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, year)))
                # do not delete zip file because it is needed for comparison
                


        # vischem and drychem
        for variable in variables_single_file:

            # compare old and new files after download
            same_files = compare_files(download_location.format(variable, variable, "allyears"), storage_location.format(variable, variable, "allyears"))

            if same_files == True:
                # delete file in temp
                os.remove(download_location.format(variable, variable, "allyears"))
            else:
                # delete old file, move new file
                os.remove(storage_location.format(variable, variable, "allyears"))
                os.rename(download_location.format(variable, variable, "allyears"), storage_location.format(variable, variable, "allyears"))

                # unzip
                with zipfile.ZipFile(storage_location.format(variable, variable, "allyears"), 'r') as zip_ref:
                    zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, "allyears")))
                os.remove(storage_location.format(variable, variable, "allyears"))
                
                
    # unzip
    elif mode == "all":
        for variable in variables_per_year:
            for year in years:
                try:
                    with zipfile.ZipFile(storage_location.format(variable, variable, year), 'r') as zip_ref:
                        zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, year)))
                    os.remove(storage_location.format(variable, variable, year))
                except:
                    print("No data to extract in {}".format(year))
            

        for variable in variables_single_file:
            with zipfile.ZipFile(storage_location.format(variable, variable, "allyears"), 'r') as zip_ref:
                zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, "allyears")))
            os.remove(storage_location.format(variable, variable, "allyears"))
            



    """driver = webdriver.Chrome()
    driver.get(url)
    page_source = driver.page_source
    driver.quit()

    # Parse the HTML using BeautifulSoup
    soup = BeautifulSoup(page_source, 'html.parser')


    for link in soup.find_all('a'):
        #print(link.get('href'))
        href = link.get('href')

        # wet deposition
        if ('wet_concentration.zip' in href):
            filename = href
            download_url = url + '/' + href
            download_file(download_url, download_location+filename)
            print(href + " downloaded successfully")

        # hourly gas, later change file location 
        if ('hourly_gas_2024' in href):
            filename = href
            download_url = url + '/' + href
            download_file(download_url, download_location+filename)
            print(href + " downloaded successfully")"""