from selenium import webdriver
from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By

from bs4 import BeautifulSoup
import requests
import time
from datetime import date
from datetime import timedelta
import pandas as pd
import os.path
import urllib
import time
import ssl 
import zipfile
import shutil
import os


def download_file(component, year, download_location, component_element):
    # add user agent to make scraper look like real user; if Forbidden403 appears again, add more fake agents and choose different one for every request
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    # create download directory
    os.makedirs(download_location+component+'/'+year+'/', exist_ok=True)

    component_link = component_element.get_attribute("href")
    component_file_name = component_element.get_attribute("text")

    # download
    r = requests.get(component_link, headers=Headers)
    if r.status_code == 200:

        with open(download_location+component+'/'+year+'/'+component_file_name, 'wb') as outfile:
            outfile.write(r.content)
        print('Downloaded {} in {}'.format(component, year))

        # unzip with shell to have option to extract in current directory; -o overwrite, -q quiet extraction no printing, -j extract in same directory
        os.system("unzip -o -q -j " +download_location+component+'/'+year+'/'+component_file_name+' -d '+download_location+component+'/'+year+'/')
        os.remove(download_location+component+'/'+year+'/'+component_file_name)
        
    elif r.status_code == 404:
        print("No data for {} in {} in integrated data".format(component, year))

    else:
        print('Problem for {} in {}'.format(component, year))


def scraper(mode, version):

    baseurl_con = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/national-air-pollution-surveillance-naps-program/Data-Donnees/{}/ContinuousData-DonneesContinu/HourlyData-DonneesHoraires/{}_{}.csv'
    baseurl_int = 'https://data-donnees.az.ec.gc.ca/data/air/monitor/national-air-pollution-surveillance-naps-program/Data-Donnees/{}/IntegratedData-DonneesPonctuelles/?lang=en'

    # wetdep
    baseurl_ions = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/monitoring-of-atmospheric-precipitation-chemistry/major-ions/AtmosphericPrecipitationChemistry-MajorIons-APQMP-AllSites-{}.csv'


    os.makedirs("/esarchive/obs/ghost/CANADA_NAPS/original_files/{}/".format(version), exist_ok=True)
    download_location = "/esarchive/obs/ghost/CANADA_NAPS/original_files/{}/".format(version)
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    # set up driver
    options = Options()
    options.add_argument("--disable-dev-shm-usage")
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")
    options.add_argument("--disable-gpu")
    options.add_argument("--disable-extensions")
    options.add_argument("user-agent={Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148}")
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)

    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)


    components_con = ["CO", "NO", "NO2", "O3", "PM10", "PM2.5", "SO2"]
    components_int = ["Carbonyls", "VOC", "IntegratedPM2.5", "IntegratedPM2.5-10", "NAPSReferenceMethodPM2.5", "PartisolPM2.5", "PMSpeciation", "PMDICHOT"]
    #components_wetdep = ["anthropogenic-organic-pollutants-in-precipitation", "major-ions", "metals-in-precipitation"]

    bdate = date(2010, 1,1) #date(1975, 1, 1) #date(1960, 1, 1) # date before record starts
    edate = date(2015,1,1) #date.today() 
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    print(years)

    for year in years:
        
        # continuous components =======================================================================
        for component in components_con:

            url = baseurl_con.format(year, component, year)
            r = requests.get(url, timeout=120, headers=Headers)

            if r.status_code == 200:

                # create download directory
                os.makedirs(download_location+component+'/', exist_ok=True)

                with open(download_location+component+'/'+component+"_"+year+".csv", 'wb') as outfile:
                    outfile.write(r.content)

                print('Downloaded {} in {}'.format(component, year))

            elif r.status_code == 404:
                print("No data for {} in {}".format(component, year))

            else:
                print('Problem for {} in {}'.format(component, year))

            time.sleep(1)


        # integrated components =======================================================================
        response = requests.get(baseurl_int.format(year), headers=Headers)
        time.sleep(2)

        if response.status_code == 404: # filters for integrated data
            print("No integrated data in {}".format(year))

        else:
            # loop integrated components
            for component in components_int:

                # open working url
                driver.get(baseurl_int.format(year))
                time.sleep(2)

                # locate elements
                try:
                    component_element = driver.find_element(By.PARTIAL_LINK_TEXT, component)
                    print("lower case " + component)
                    download_file(component, year, download_location, component_element)

                except:
                    try:
                        component_element = driver.find_element(By.PARTIAL_LINK_TEXT, component.upper())
                        print("upper case "+component)
                        download_file(component, year, download_location, component_element)


                    except:
                        print("No file found for {} in {}".format(component, year))


        # wet deposition ===============================================================================
        res = requests.get(baseurl_ions.format(year), headers=Headers, timeout=120)
        time.sleep(2)

        if res.status_code == 200:

            # create download directory
            os.makedirs(download_location+"precip/major-ions/", exist_ok=True)

            with open(download_location+"precip/major-ions/"+os.path.basename(baseurl_ions.format(year)), 'wb') as outfile:
                outfile.write(res.content)

            print('Downloaded {}'.format(os.path.basename(baseurl_ions.format(year))))
        
        elif res.status_code == 404:
            print("No major ions data in {}".format(year))
        
        else:
            print("Problem with major ions download")


    # metals and organics
    url_metals = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/monitoring-of-atmospheric-precipitation-chemistry/metals-in-precipitation/AtmosphericPrecipitationChemistry-MetalsInPrecipitation-GLB-MultipleSites-1993_2017.csv'
    r = requests.get(url_metals, timeout=120, headers=Headers)

    if r.status_code == 200:

        # create download directory
        os.makedirs(download_location+"precip/metals/", exist_ok=True)

        with open(download_location+"precip/metals/"+os.path.basename(url_metals), 'wb') as outfile:
                outfile.write(res.content)

        print('Downloaded {}'.format(os.path.basename(url_metals)))
    else:
        print("Problem with metals download")


    url_organics = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/monitoring-of-atmospheric-precipitation-chemistry/anthropogenic-organic-pollutants-in-precipitation/AtmosphericPrecipitationChemistry-AOPP_{}-GLBM-MultipleSites-{}_{}.csv'

    components_org = ["FR", "OC", "PAH", "PCB"]
    year_start = ["2002", '1990', '1990', '2002']
    year_end = ['2015', '2015', '2015', '2016']

    # create download directory
    os.makedirs(download_location+"precip/organic/", exist_ok=True)

    for i in range(len(components_org)):

        r = requests.get(url_organics.format(components_org[i], year_start[i], year_end[i]), timeout=120, headers=Headers)

        if r.status_code == 200:

            with open(download_location+"precip/organic/"+os.path.basename(url_organics.format(components_org[i], year_start[i], year_end[i])), 'wb') as outfile:
                    outfile.write(r.content)

            print('Downloaded {}'.format(url_organics.format(components_org[i], year_start[i], year_end[i])))
        else:
            print("Problem with organics download")


    driver.close()

