from selenium import webdriver
from bs4 import BeautifulSoup
import requests
import hashlib
from datetime import date
from datetime import timedelta
import pandas
import os.path
import urllib
import time
import ssl 
import zipfile
from compare_two_files import compare_files

from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options





def scraper(mode, version):

    
    # find cities: scraping difficult, maybe do it later
    """url = 'https://www.airnow.gov/international/us-embassies-and-consulates/'
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc)
    driver.get(url)
    time.sleep(5)

    html = driver.page_source
    soup = BeautifulSoup(html, features="html.parser")
    div = soup.find_all('select')
    print(div)


    driver.close()"""
    
    cities = ['NewDehli', 'Chennai', 'Kolkata', 'Mumbai', 'Hyderabad', 'JakartaCentral', 'JakartaSouth', 'Ulaanbaatar', 'HoChiMinhCity', 'Hanoi', 'Bogota', 'Lima', 'AddisAbabaCentral', 'AddisAbabaJacros', 'Kampala', 'Beijing', 'Shanghai', 'Guangzhou', 'Chengdu', 'Shenyang', 'Dhaka', 'AbuDhabi', 'Dubai', 'Pristina', 'KuwaitCity', 'Manama', 'Ashgabat', 'EmbassyKathmandu', 'PhoraDurbarKathmandu', 'Colombo', 'Algiers', 'BuenosAires', 'Sarajevo', 'BanjaLuka', 'Curacao', 'Dhahran', 'Jeddah', 'Riyadh', 'Astana', 'Almaty', 'Tashkent', 'Baghdad', 'Rangoon', 'Bishkek', 'Dushanbe', 'SanJose', 'Islamabad', 'Lahore', 'Karachi', 'Peshawar', 'Amman', 'Vientiane', 'Bamako', 'Antananarivo', 'GuatemalaCity', 'Libreville', 'Abidjan', 'Chisinau', 'Conakry', 'NDjamena', 'Accra', 'KualaLumpur', 'Lagos', 'Abuja', 'Doha', 'Nairobi', 'Baku', 'Cairo', 'Dakar', 'Tijuana', 'Yerevan', 'Kinshasa', 'Ouagadougou', 'SaoPaulo', 'Kigali', 'Lome', 'Asuncion', 'Belmopan', 'Manila', 'Maputo']
    variables = ["PM2.5", "OZONE"]
    

    base_url = "https://dosairnowdata.org/dos/historical/"


    if mode == 'all':
        bdate = date(1980, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)

        os.makedirs('/esarchive/obs/ghost/US_EPA_AirNow_DOS/original_files/{}/'.format(version), exist_ok=True)

        download_location = '/esarchive/obs/ghost/US_EPA_AirNow_DOS/original_files/'+version+'/{}_{}_{}_YTD.csv'
        storage_location = '/esarchive/obs/ghost/US_EPA_AirNow_DOS/original_files/'+version+'/{}_{}_{}_YTD.csv'

    elif mode == 'nrt':
        bdate = date.today() 
        edate = date.today() + timedelta(days = 365)
        download_location = '/esarchive/obs/ghost/US_EPA_AirNow_DOS/original_files/nrt/temp/{}_{}_{}_YTD.csv'
        storage_location = '/esarchive/obs/ghost/US_EPA_AirNow_DOS/original_files/nrt/{}_{}_{}_YTD.csv'


    else:
        print('time mode inapplicable')

    # create date array, per year
    years = pandas.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()

    print(years)

    for city in cities:

        for variable in variables:

            for year in years:

                url = '{}{}/{}/{}_{}_{}_YTD.csv'.format(base_url, city, year, city, variable, year)
                r = requests.get(url, timeout=120)

                if r.status_code == 200:
                    urllib.request.urlretrieve(url, download_location.format(city, variable, year))
                    print('Downloaded {}'.format(url))

                    # compare old and new file
                    if mode == "nrt":

                        same_files = compare_files(download_location.format(city, variable, year), storage_location.format(city, variable, year))

                        if same_files == True:
                            # delete file in temp
                            os.remove(download_location.format(city, variable, year))
                        else:
                            # delete old file, move new file
                            os.remove(storage_location.format(city, variable, year))
                            os.rename(download_location.format(city, variable, year), storage_location.format(city, variable, year))


                else:
                    print('No {} {} in {}'.format(variable, year, city))

                time.sleep(1)