import requests
from datetime import date
from datetime import timedelta
import pandas
import os.path
import urllib
import time
import ssl 
import zipfile
from compare_two_files import compare_files
import csv
import json
import shutil
import metadata_parsing_functions


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    ssl._create_default_https_context = ssl._create_unverified_context
    
    variables_per_year = ["ozone", "hourly_gas"] 
    variables_single_file = ["drychem", "vischem"]

    base_url = "https://gaftp.epa.gov/castnet/CASTNET_Outgoing/data/"
    os.makedirs('{}/US_EPA_CASTNET/original_files/{}/'.format(dl_root, version), exist_ok=True)

    if mode == 'all':
        bdate = date(1980, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)
        download_location = dl_root+'/US_EPA_CASTNET/original_files/'+version+'/{}/{}_{}.zip'
        storage_location = dl_root+'/US_EPA_CASTNET/original_files/'+version+'/{}/{}_{}.zip'

    elif mode == 'nrt':
        bdate = start
        edate = stop + timedelta(days = 365)
        download_location = dl_root+'/US_EPA_CASTNET/original_files/nrt/{}/temp/{}_{}.zip'
        storage_location = dl_root+'/US_EPA_CASTNET/original_files/nrt/{}/{}_{}.zip'
        version = mode

    else:
        print('time mode inapplicable')

    # create date array, per year
    years = pandas.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    
    # ozone and hourly gas
    for variable in variables_per_year:

        os.makedirs('{}/US_EPA_CASTNET/original_files/{}/{}/'.format(dl_root, version, variable), exist_ok=True)
        if mode == 'nrt':
            os.makedirs('{}/US_EPA_CASTNET/original_files/{}/{}/temp/'.format(dl_root, version, variable), exist_ok=True)

        for year in years:

            url = '{}{}_{}.zip'.format(base_url, variable, year)
            n_tries = 0
            errcode = 999

            while (n_tries < n_max_tries) and (errcode != 200):
                r = requests.get(url, timeout=max_time_per_dl, verify=False)

                if r.status_code == 200:
                    with open(storage_location.format(variable, variable, year), 'wb') as file:
                        file.write(r.content)
                    urllib.request.urlretrieve(url, storage_location.format(variable, variable, year))
                    print('Downloaded {}'.format(url))

                    # unzip
                    with zipfile.ZipFile(storage_location.format(variable, variable, year), 'r') as zip_ref:
                        zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, year)))
                    os.remove(storage_location.format(variable, variable, year))

                    errcode = r.status_code
                elif r.status_code == 404:
                    print('No {}'.format(url))
                    errcode = 200 #skip
                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    time.sleep(n_tries ** 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
            
            time.sleep(1)



    # drychem and vichem
    for variable in variables_single_file:

        os.makedirs('{}/US_EPA_CASTNET/original_files/{}/{}/'.format(dl_root, version, variable), exist_ok=True)
        if mode == 'nrt':
            os.makedirs('{}/US_EPA_CASTNET/original_files/{}/{}/temp/'.format(dl_root, version, variable), exist_ok=True)

        url = '{}{}.zip'.format(base_url, variable)
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(url, timeout=max_time_per_dl, verify=False)

            if r.status_code == 200:
                with open(storage_location.format(variable, variable, "allyears"), 'wb') as file:
                    file.write(r.content)
                #urllib.request.urlretrieve(url, storage_location.format(variable, variable, "allyears"))
                print('Downloaded {}'.format(url))

                # unzip
                with zipfile.ZipFile(storage_location.format(variable, variable, "allyears"), 'r') as zip_ref:
                    zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, "allyears")))
                os.remove(storage_location.format(variable, variable, "allyears"))

                errcode = r.status_code

            elif r.status_code == 404:
                print('No {}'.format(url))
                errcode = 200 #skip
            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
            
        time.sleep(1)


    # compare old and new file
    # if mode == 'nrt':
    #     print('hola')
    #     for variable in variables_per_year:
    #         print(variable)
    #         # compare old and new files after download
    #         same_files = compare_files(download_location.format(variable, variable, year), storage_location.format(variable, variable, year))
    #         print(same_files)

    #         if same_files == True:
    #             # delete file in temp
    #             os.remove(download_location.format(variable, variable, year))
    #         else:
    #             # delete old file, move new file
    #             os.remove(storage_location.format(variable, variable, year))
    #             os.rename(download_location.format(variable, variable, year), storage_location.format(variable, variable, year))

    #             # unzip
    #             with zipfile.ZipFile(storage_location.format(variable, variable, year), 'r') as zip_ref:
    #                 zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, year)))
    #                 # do not delete zip file because it is needed for comparison
                


    #     # vischem and drychem
    #     for variable in variables_single_file:

    #         # compare old and new files after download
    #         same_files = compare_files(download_location.format(variable, variable, "allyears"), storage_location.format(variable, variable, "allyears"))

    #         if same_files == True:
    #             # delete file in temp
    #             os.remove(download_location.format(variable, variable, "allyears"))
    #         else:
    #             # delete old file, move new file
    #             os.remove(storage_location.format(variable, variable, "allyears"))
    #             os.rename(download_location.format(variable, variable, "allyears"), storage_location.format(variable, variable, "allyears"))

    #             # unzip
    #             with zipfile.ZipFile(storage_location.format(variable, variable, "allyears"), 'r') as zip_ref:
    #                 zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, "allyears")))
    #             os.remove(storage_location.format(variable, variable, "allyears"))
                
                
    # unzip
    # if mode == "all":
    #     for variable in variables_per_year:
    #         for year in years:
    #             try:
    #                 with zipfile.ZipFile(storage_location.format(variable, variable, year), 'r') as zip_ref:
    #                     zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, year)))
    #                 os.remove(storage_location.format(variable, variable, year))
    #             except:
    #                 print("No data to extract in {}".format(year))
            

    #     for variable in variables_single_file:
    #         with zipfile.ZipFile(storage_location.format(variable, variable, "allyears"), 'r') as zip_ref:
    #             zip_ref.extractall(os.path.dirname(storage_location.format(variable, variable, "allyears")))
    #         os.remove(storage_location.format(variable, variable, "allyears"))


    # download WET DEPOSITION data ===============================================================================
    os.makedirs('{}/US_EPA_CASTNET/original_files/{}/{}/temp/'.format(dl_root, version, 'wetdep'), exist_ok=True)
    
    urls = {'https://gaftp.epa.gov/castnet/CASTNET_Outgoing/data/total_deposition_composition.zip': 'total_dep',
            'https://gaftp.epa.gov/castnet/CASTNET_Outgoing/data/wet_concentration.zip': 'wet_conc',
            'https://gaftp.epa.gov/castnet/CASTNET_Outgoing/data/wetchem.zip': 'wet_chem'} 

    for url in urls:
        file_name = urls[url]
        n_tries = 0
        errcode = 999
        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(url, timeout=max_time_per_dl, verify=False)
            if r.status_code == 200:
                with open(download_location.format('wetdep', file_name, ''), 'wb') as file:
                    file.write(r.content)
                print('Downloaded {}'.format(url))

                # unzip
                with zipfile.ZipFile(download_location.format('wetdep', file_name, ''), 'r') as zip_ref:
                    zip_ref.extractall(os.path.dirname(storage_location.format('wetdep', file_name, '')))
                os.remove(download_location.format('wetdep', file_name, ''))

                errcode = r.status_code
            elif r.status_code == 404:
                print('No {}'.format(url))
                errcode = 200 #skip
            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
            



def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    ssl._create_default_https_context = ssl._create_unverified_context
    download_location = '{}/US_EPA_CASTNET/metadata/{}/network_provided/'.format(dl_root, version)
    
    today = date.today().strftime('%Y%m%d')
    url = 'https://gaftp.epa.gov/castnet/CASTNET_Outgoing/data/site.zip'
    n_tries = 0
    errcode = 999
    
    # download recent metadata file
    while (n_tries < n_max_tries) and (errcode != 200):
        r = requests.get(url, timeout=max_time_per_dl, verify=False)

        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location+'CASTNET_META_{}.zip'.format(today))
            print('Downloaded {}'.format(url))
            errcode = r.status_code
        elif r.status_code == 404:
            print('No {}'.format(url))
            errcode = 200 #skip
        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
    
    
    with zipfile.ZipFile(download_location+'CASTNET_META_{}.zip'.format(today), 'r') as zip_ref:
        zip_ref.extractall(os.path.dirname(download_location))
    os.remove(download_location+'CASTNET_META_{}.zip'.format(today))

    # rename file
    os.rename(download_location+'site.csv', download_location+'US_EPA_CASTNET_META_{}.csv'.format(today))


    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)
