import requests
import time
import pandas as pd
from datetime import date
from datetime import timedelta
import os.path
import urllib
import tarfile
import shutil
import gzip
import csv
import json
import metadata_parsing_functions


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    base_url = 'http://www.aire.cdmx.gob.mx/opendata/anuales_horarios_gz/contaminantes_{}.csv.gz'
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    if mode == 'all':
        bdate = date(1986, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today()

        os.makedirs('{}/MEXICO_CDMX/original_files/{}/'.format(dl_root, version), exist_ok=True)
        download_location = dl_root+'/MEXICO_CDMX/original_files/'+version+'/contaminantes_{}.csv.gz'

    elif mode == 'nrt':
        print("No nrt MEXICO_CDMX")
        exit()

    else:
        print('time mode inapplicable')


    # create date array, daily; format to YYYYMMDD
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()

    # download
    for year in years:
        url = base_url.format(year)
        n_tries = 0
        errcode = 999
        try: 
            while (n_tries < n_max_tries) and (errcode != 200):
                r = requests.get(url, timeout=max_time_per_dl, headers=Headers)

                if r.status_code == 200:
                    with open(download_location.format(year), 'wb') as f:
                        f.write(r.content)
                    urllib.request.urlretrieve(url, download_location.format(year))
                    print('Downloaded {}'.format(url))
                    # unzip
                    with gzip.open(download_location.format(year), 'rb') as f_in:
                        with open(download_location.format(year)[:-3], 'wb') as f_out:
                            shutil.copyfileobj(f_in, f_out)
                    # remove files
                    os.remove(download_location.format(year))

                    errcode = r.status_code
                elif r.status_code == 404:
                    print("No data found, error 404, year {}".format(year))
                    errcode = 200
                elif r.status_code == 403:
                    print("Permission denied for {}".format(year))
                    errcode = 200
                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    max_time_per_dl = max_time_per_dl*2 # increase waiting time 
                    time.sleep(n_tries ** 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))

        except:
            pass
        time.sleep(10)


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    url_metadata = 'http://www.aire.cdmx.gob.mx/opendata/catalogos/cat_estacion.csv'
    download_location = dl_root+"/MEXICO_CDMX/metadata/{}/network_provided/MEXICO_CDMX_META_{}.csv"

    n_tries = 0
    errcode = 999
    today = date.today()
    
    while (n_tries < n_max_tries) and (errcode != 200):
        r = requests.get(url_metadata, timeout=max_time_per_dl)
        if r.status_code == 200:
            with open(download_location.format(version, '_unformatted'), 'wb') as outfile:
                outfile.write(r.content)
            print('Downloaded metadata')
            errcode = r.status_code
        elif r.status_code == 404:
            print("No metadata found, error 404")
            errcode = 200
        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            max_time_per_dl = max_time_per_dl*2 # increase waiting time 
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_metadata, n_tries, max_time_per_dl, errcode))
    time.sleep(1)

    # import it as pandas to clean header
    meta_file = pd.read_csv(download_location.format(version, '_unformatted'), header=[1], encoding='ISO-8859-1')
    meta_file.to_csv(download_location.format(version, today.strftime('%Y%m%d')), index=False)
    os.remove(download_location.format(version, '_unformatted'))


    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)
    

    # create json from original metadata file
    """json_metadata = {}
    with open('{}/MEXICO_CDMX/metadata/{}/network_provided/MEXICO_CDMX_META.csv'.format(dl_root, version), 'r', encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['cve_estac']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata[key] = row

    with open('{}/MEXICO_CDMX/metadata/{}/processed/MEXICO_CDMX_META.json'.format(dl_root, version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4))
        
    
    
    # create json in desired shape from current metadata file
    json_metadata_now = {}  
    with open(download_location.format(version, today.strftime('%Y%m%d')), encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['cve_estac']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata_now[key] = row

    
    # read standardised file to compare!
    with open('{}/MEXICO_CDMX/metadata/{}/processed/MEXICO_CDMX_META.json'.format(dl_root, version), 'r', encoding='ISO-8859-1') as f:
        json_metadata = json.loads(f.read())

        for station in json_metadata: # loop through all the old stations
            if station in json_metadata_now.keys(): # if station is in current meta data, go on
                for parameter in json_metadata[station]:
                    if parameter in json_metadata_now[station].keys(): # check if column of csv exists in new file
                        if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                            # if different value, append the standardised metadeta file
                            print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                            json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                            json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                        else:
                            pass
                    else:
                        print('{} not in new metadata file'.format(parameter))
            else:
                print('Station {} was abolished'.format(station))

        for station in json_metadata_now: # loop through all the new stations
            for parameter in json_metadata_now[station]: # loop through all the parameters
                if station in json_metadata.keys(): # if station is in old meta data
                    pass # comparison was done before
                else: # new station appeared!
                    print('New station {}'.format(station))
                    json_metadata.update({station: json_metadata_now[station]})
                # is there a new parameter that wasn't in the old file?
                if parameter in json_metadata[station].keys():
                    pass # parameter (column) is already there
                else:
                    print('{} is new'.format(parameter))
                    json_metadata[station].update({parameter: json_metadata_now[station][parameter]})


    # safe
    with open('{}/MEXICO_CDMX/metadata/{}/processed/MEXICO_CDMX_META.json'.format(dl_root, version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4))"""