from selenium import webdriver
from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By

import requests
import time
from datetime import date
from datetime import timedelta
import pandas as pd
import os.path
import time
import os
import csv
import json
import metadata_parsing_functions


def download_file(component, year, download_location, component_element, n_max_tries, max_time_per_dl):
    # add user agent to make scraper look like real user; if Forbidden403 appears again, add more fake agents and choose different one for every request
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    # create download directory
    os.makedirs(download_location+component+'/'+year+'/', exist_ok=True)

    component_link = component_element.get_attribute("href")
    component_file_name = component_element.get_attribute("text")


    # download
    n_tries = 0
    errcode = 999

    while (n_tries < n_max_tries) and (errcode != 200):
        r = requests.get(component_link, headers=Headers, timeout=max_time_per_dl)
        if r.status_code == 200:

            with open(download_location+component+'/'+year+'/'+component_file_name, 'wb') as outfile:
                outfile.write(r.content)
            print('Downloaded {} in {}'.format(component, year))

            # unzip with shell to have option to extract in current directory; -o overwrite, -q quiet extraction no printing, -j extract in same directory
            os.system("unzip -o -q -j " +download_location+component+'/'+year+'/'+component_file_name+' -d '+download_location+component+'/'+year+'/')
            os.remove(download_location+component+'/'+year+'/'+component_file_name)
            errcode = r.status_code

        elif r.status_code == 404:
            print("No data for {} in {} in integrated data".format(component, year))
            errcode = 200

        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(component_link, n_tries, max_time_per_dl, errcode))


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    baseurl_con = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/national-air-pollution-surveillance-naps-program/Data-Donnees/{}/ContinuousData-DonneesContinu/HourlyData-DonneesHoraires/{}_{}.csv'
    baseurl_int = 'https://data-donnees.az.ec.gc.ca/data/air/monitor/national-air-pollution-surveillance-naps-program/Data-Donnees/{}/IntegratedData-DonneesPonctuelles/?lang=en'

    # wetdep
    baseurl_ions = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/monitoring-of-atmospheric-precipitation-chemistry/major-ions/AtmosphericPrecipitationChemistry-MajorIons-APQMP-AllSites-{}.csv'


    os.makedirs("{}/CANADA_NAPS/original_files/{}/".format(dl_root,version), exist_ok=True)
    download_location = "{}/CANADA_NAPS/original_files/{}/".format(dl_root,version)
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}
    Headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Cache-Control": "max-age=0",}

    # set up driver
    options = Options()
    options.add_argument("--disable-dev-shm-usage")
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")
    options.add_argument("--disable-gpu")
    options.add_argument("--disable-extensions")
    options.add_argument("user-agent={Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148}")
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)

    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    components_con = ["CO", "NO", "NO2", "O3", "PM10", "PM2.5", "SO2"]
    components_int = ["Carbonyls", "VOC", "IntegratedPM2.5", "IntegratedPM2.5-10", "NAPSReferenceMethodPM2.5", "PartisolPM2.5", "PMSpeciation", "PMDICHOT"]
    #components_wetdep = ["anthropogenic-organic-pollutants-in-precipitation", "major-ions", "metals-in-precipitation"]

    bdate = date(1974, 1,1) # date before record starts
    edate = date.today() + timedelta(days=365)
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    print(years)

    for year in years:
        
        # continuous components =======================================================================
        for component in components_con:

            url = baseurl_con.format(year, component, year)
            r = requests.get(url, timeout=max_time_per_dl, headers=Headers)
            n_tries = 0
            errcode = 999

            while (n_tries < n_max_tries) and (errcode != 200):

                if r.status_code == 200:
                    # create download directory
                    os.makedirs(download_location+component+'/', exist_ok=True)
                    with open(download_location+component+'/'+component+"_"+year+".csv", 'wb') as outfile:
                        outfile.write(r.content)
                    print('Downloaded {} in {}'.format(component, year))
                    errcode = r.status_code

                elif r.status_code == 404:
                    print("No data for {} in {}".format(component, year))
                    errcode = 200

                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    time.sleep(n_tries ** 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
            time.sleep(1)


        # integrated components =======================================================================
        response = requests.get(baseurl_int.format(year), headers=Headers, timeout=max_time_per_dl)
        time.sleep(2)

        if response.status_code == 404: # filters for integrated data
            print("No integrated data in {}".format(year))

        else:
            # loop integrated components
            for component in components_int:

                # open working url
                driver.get(baseurl_int.format(year))
                time.sleep(2)

                # locate elements
                try:
                    component_element = driver.find_element(By.PARTIAL_LINK_TEXT, component)
                    download_file(component, year, download_location, component_element)

                except:
                    try:
                        component_element = driver.find_element(By.PARTIAL_LINK_TEXT, component.upper())
                        download_file(component, year, download_location, component_element)

                    except:
                        print("No file found for {} in {}".format(component, year))


        # wet deposition ===============================================================================
        res = requests.get(baseurl_ions.format(year), headers=Headers, timeout=max_time_per_dl)
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):

            if res.status_code == 200:
                # create download directory
                os.makedirs(download_location+"precip/major-ions/", exist_ok=True)

                with open(download_location+"precip/major-ions/"+os.path.basename(baseurl_ions.format(year)), 'wb') as outfile:
                    outfile.write(res.content)

                print('Downloaded {}'.format(os.path.basename(baseurl_ions.format(year))))
                errcode = res.status_code

            elif res.status_code == 404:
                print("No major ions data in {}".format(year))
                errcode = 200

            else:
                # try again
                print('Response error {}, attempt {}'.format(res.status_code, n_tries))
                errcode = res.status_code
                n_tries += 1
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(baseurl_ions.format(year), n_tries, max_time_per_dl, errcode))
        time.sleep(1)


    # metals and organics
    url_metals = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/monitoring-of-atmospheric-precipitation-chemistry/metals-in-precipitation/AtmosphericPrecipitationChemistry-MetalsInPrecipitation-GLB-MultipleSites-1993_2017.csv'
    r = requests.get(url_metals, timeout=max_time_per_dl, headers=Headers)
    n_tries = 0
    errcode = 999

    while (n_tries < n_max_tries) and (errcode != 200):
        if r.status_code == 200:

            # create download directory
            os.makedirs(download_location+"precip/metals/", exist_ok=True)

            with open(download_location+"precip/metals/"+os.path.basename(url_metals), 'wb') as outfile:
                outfile.write(r.content)
            print('Downloaded {}'.format(os.path.basename(url_metals)))
            errcode = r.status_code

        elif r.status_code == 404:
            print("No metal data found, error 404")
            errcode = 200
        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_metals, n_tries, max_time_per_dl, errcode))
    time.sleep(1)



    url_organics = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/monitoring-of-atmospheric-precipitation-chemistry/anthropogenic-organic-pollutants-in-precipitation/AtmosphericPrecipitationChemistry-AOPP_{}-GLBM-MultipleSites-{}_{}.csv'

    components_org = ["FR", "OC", "PAH", "PCB"]
    year_start = ["2002", '1990', '1990', '2002']
    year_end = ['2015', '2015', '2015', '2016']

    # create download directory
    os.makedirs(download_location+"precip/organic/", exist_ok=True)

    for i in range(len(components_org)):
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(url_organics.format(components_org[i], year_start[i], year_end[i]), timeout=max_time_per_dl, headers=Headers)

            if r.status_code == 200:
                with open(download_location+"precip/organic/"+os.path.basename(url_organics.format(components_org[i], year_start[i], year_end[i])), 'wb') as outfile:
                        outfile.write(r.content)
                print('Downloaded {}'.format(url_organics.format(components_org[i], year_start[i], year_end[i])))
                errcode = r.status_code

            elif r.status_code == 404:
                print("No organics data found, error 404")
                errcode = 200
            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_organics, n_tries, max_time_per_dl, errcode))
        time.sleep(1)


    driver.close()


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    network = os.path.basename(__file__).split('_downloa')[0]
    identifier = meta_params['identifier']

    url_metadata = 'https://data-donnees.az.ec.gc.ca/api/file?path=/air/monitor/national-air-pollution-surveillance-naps-program/ProgramInformation-InformationProgramme/StationsNAPS-StationsSNPA.csv'
    download_location = dl_root+"/CANADA_NAPS/metadata/{}/network_provided/CANADA_NAPS_META_{}.csv"
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}
    r = requests.get(url_metadata, headers=Headers, timeout=max_time_per_dl)
    n_tries = 0
    errcode = 999
    today = date.today()
    
    while (n_tries < n_max_tries) and (errcode != 200):

        if r.status_code == 200:

            with open(download_location.format(version, '_unformatted'), 'wb') as outfile:
                outfile.write(r.content)

            print('Downloaded metadata')
            errcode = r.status_code

        elif r.status_code == 404:
            print("No metadata found, error 404")
            errcode = 200

        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_metadata, n_tries, max_time_per_dl, errcode))
    time.sleep(1)

    # import it as pandas to clean header
    meta_file = pd.read_csv(download_location.format(version, '_unformatted'), header=[4])
    meta_file = meta_file[1 : meta_file.index[meta_file[identifier] == '**********Definitions_Définitions**********'][0]] # cut out french line and bottom
    meta_file.to_csv(download_location.format(version, today.strftime('%Y%m%d')), index=False)
    os.remove(download_location.format(version, '_unformatted'))

    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)

