import os
import sys
import requests
import logging
import urllib
import datetime as dt
import json
import csv
import zipfile
import time
import pandas as pd
from pathlib import Path
from datetime import date
from datetime import timedelta
import shutil
import metadata_parsing_functions

#from convert_json_csv import convert_json_csv

class aqs_epa_api(object):
    """defines and configures API to query the aqms database"""

    def __init__(self, ):
        self.logger = logging.getLogger(__file__)
        self.url_api = "https://aqs.epa.gov/data/api/"
        self.headers = {'content-type': 'application/json', 'accept': 'application/json'}

        self.get_states = 'list/states'
        self.get_counties = 'list/countiesByState'
        self.get_sites = 'list/sitesByCounty'

        self.get_parameters = "list/parametersByClass"
        self.get_data_by_site = 'sampleData/bySite'
        self.get_Dailydata_by_site = 'dailyData/bySite'

        self.get_Annualdata_by_state = 'annualData/byState'

        return
    
    def get_parameters_in_list(self, ):
        """build a query to return data by state"""
        query = urllib.parse.urljoin(self.url_api, self.get_parameters)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'pc': 'ALL'}
        response = requests.get(url=query, data='', params=params)
        return response.text

    def states(self, ):
        """build a query to return states in list"""
        query = urllib.parse.urljoin(self.url_api, self.get_states)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34'}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return print('Response error')
    
    def counties(self, code):
        """build a query to return list of counties by state"""
        query = urllib.parse.urljoin(self.url_api, self.get_counties)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'state': str(code)}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return 'Response error'

    def sites(self, code, ccode):
        """build a query to return list of sites per county"""
        query = urllib.parse.urljoin(self.url_api, self.get_sites)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'state': str(code), 'county': str(ccode)}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return 'Response error'

    def request_data(self, n_max_tries, max_time_per_dl, query, params):
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            response = requests.get(url=query, data='', params=params, timeout=max_time_per_dl)
                    
            if response.status_code == 200:
                return response.text
                errcode = response.status_code
            elif response.status_code == 400: # bad request error
                print('Error 400 bad request')
                errcode = 200
                
            else:
                # try again
                print('Response error {}, attempt {}'.format(response.status_code, n_tries))
                errcode = response.status_code
                n_tries += 1

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(params, n_tries, max_time_per_dl, errcode))
            return None


    def get_data_site(self, code, ccode, scode, pcode, bbdate, eedate):
        """build a query to return data by site"""
        query = urllib.parse.urljoin(self.url_api, self.get_data_by_site)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'param': pcode, 'bdate': bbdate, 'edate': eedate, 'state': str(code), 'county': str(ccode), 'site': str(scode)}
        return query, params


    def get_Dailydata_site(self, code, ccode, scode, pcode, bbdate, eedate):
        """build a query to return data by site"""
        query = urllib.parse.urljoin(self.url_api, self.get_Dailydata_by_site)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'param': pcode, 'bdate': bbdate, 'edate': eedate, 'state': str(code), 'county': str(ccode), 'site': str(scode)}
        return query, params


    def get_Annualdata_state(self, code, pcode, bbdate, eedate):
        """build a query to return data by state per year"""
        query = urllib.parse.urljoin(self.url_api, self.get_Annualdata_by_state)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'param': pcode, 'bdate': bbdate, 'edate': eedate, 'state': str(code)}
        return query, params
    


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    # copy ghost standards in current directory for easier import
    shutil.copy('/esarchive/scratch/rgrodofz/software/ghost/GHOST/processing_modules/GHOST_standards.py', '/esarchive/scratch/rgrodofz/software/ghost/GHOST/download_scripts/GHOST_standards.py')
    from GHOST_standards import standard_parameters

    if mode == 'all':
        bdate = date(1970, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)

        os.makedirs('{}/US_EPA_AQS/original_files/{}/'.format(dl_root, version), exist_ok=True)

    elif mode == 'nrt':
        version = mode
        bdate = start
        edate = stop 

        os.makedirs('{}/US_EPA_AQS/original_files/nrt/'.format(dl_root), exist_ok=True)
        download_location = '{}/US_EPA_AQS/original_files/nrt/'.format(dl_root)

    else:
        print('time mode inapplicable')


    time_resolutions = ['hourly', 'daily']
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    print(years)

    components = {}
    for standard_parameter in standard_parameters:
        aqs_code = standard_parameters[standard_parameter]['aqs_code']
        if aqs_code: # check if aqs_code is available
            components[standard_parameter] = standard_parameters[standard_parameter]['aqs_code']


    if mode == 'all': # use US EPA AQS API

        # params_list = json.loads(aqs_epa_api().get_parameters_in_list())['Data']
        # with open('US_EPA_AQS_parameters.json', 'w') as f:
        #     f.write(json.dumps(params_list, indent=4))
        # params_list = pd.DataFrame(params_list)
        # params_list.to_csv('/esarchive/scratch/rgrodofz/software/ghost/GHOST/download_scripts/US_EPA_AQS_parameterlist.csv', index=False)


        # find all states
        states = json.loads(aqs_epa_api().states())['Data']
        for state in states:
            code = state.get('code')
            state_name = state.get('value_represented')
            print('Downloading... '+state_name)

            for component in components:
                pcode = components[component][0] # select first aqs_code if there are multiple

                for year in years:
                    # yearly files
                    bbdate = date(int(year), 1, 1).strftime('%Y%m%d')
                    eedate = date(int(year), 12, 31).strftime('%Y%m%d')

                    # find available data per state
                    query, params = aqs_epa_api().get_Annualdata_state(code, pcode, bbdate, eedate)

                    # find out if there is data 
                    try:
                        data_per_state = aqs_epa_api().request_data(n_max_tries, max_time_per_dl, query, params)
                    except Exception as e:
                        print(type(e).__name__, e, code, pcode, year)
                        continue

                    if data_per_state == None:
                        continue # skip

                    # check if data is available for that state
                    if json.loads(data_per_state)['Header'][0]['status'] == 'Success':

                        yearly_datapoints = json.loads(data_per_state)['Data']

                        for yearly_datapoint in yearly_datapoints: # loop through entries in json that represent yearly average of data

                            if yearly_datapoint["sample_duration"] == "1 HOUR": 
                                # this does not indicate hourly sample resolution! I use it to check if there is any data available that used to be sampled hourly.
                                # However, "1 HOUR" means here daily maximum of hourly sampled data. We do not download this but just use it to check if data is available
                                # Assume that if there is hourly data, there is also daily data. Download both.
                                print("There is hourly data in {} for {} in {}".format(state_name, pcode, year))
                                
                                # county code and site code
                                ccode = yearly_datapoint['county_code']
                                scode = yearly_datapoint['site_number']

                                if scode is not None:

                                    for time_resolution in time_resolutions:

                                        if time_resolution == 'hourly':
                                            query, params = aqs_epa_api().get_data_site(code, ccode, scode, pcode, bbdate, eedate)
                                        elif time_resolution == 'daily':
                                            query, params = aqs_epa_api().get_Dailydata_site(code, ccode, scode, pcode, bbdate, eedate)
                                        else:
                                            print('Time resolution problem')

                                        # download    
                                        try:
                                            data_per_site = aqs_epa_api().request_data(n_max_tries, max_time_per_dl, query, params)
                                        except Exception as e:
                                            print(type(e).__name__, e, time_resolution, code, ccode, scode, pcode, year)
                                            continue

                                        if data_per_site == None:
                                            continue # skip if download failed

                                        # check if data is available for that site
                                        if json.loads(data_per_site)['Header'][0]['status'] == 'Success':
                                            # save files
                                            filename = '{}_{}_{}_{}_{}_{}.json'.format(time_resolution, code, ccode, scode, pcode, year)

                                            # folder per year
                                            os.makedirs('{}/US_EPA_AQS/original_files/{}/{}/{}/'.format(dl_root, version, component, year), exist_ok=True)
                                            download_location = '{}/US_EPA_AQS/original_files/{}/{}/{}/'.format(dl_root, version, component, year)

                                            with open(download_location+filename, 'w') as outfile:
                                                outfile.write(data_per_site)
                                            print("Downloaded {}".format(filename))
                                        else:
                                            print('No data found for {}_{}_{}_{}_{}_{}'.format(time_resolution, code, ccode, scode, pcode, year))
                                            continue
                            else:
                                continue


    elif mode == 'nrt': # use AirNow

        base_url = 'https://s3-us-west-1.amazonaws.com//files.airnowtech.org/airnow/{}/{}/HourlyData_{}.dat'

        # create hourly array
        hours = pd.date_range(bdate, edate, freq='H').strftime('%Y%m%d%H').tolist()[:-1]

        # download
        for hour in hours:
            url = base_url.format(hour[:4], hour[:8], hour)
            n_tries = 0
            errcode = 999

            while (n_tries < n_max_tries) and (errcode != 200):
                r = requests.get(url, timeout=max_time_per_dl)
                if r.status_code == 200:
                    with open(download_location+'HourlyData_'+hour+'.dat', 'wb') as file:
                        file.write(r.content)
                    print('Downloaded {}'.format(url))
                    errcode = r.status_code

                elif r.status_code == 404:
                    print("No data found, error 404")
                    errcode = 200

                elif r.status_code == 403:
                    print("Permission denied for {}".format(url))
                    errcode = 200

                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    max_time_per_dl = max_time_per_dl*2
                    time.sleep(n_tries ** 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))


    # delete GHOST_standards.py in download_script directory after download is finished
    os.remove('/esarchive/scratch/rgrodofz/software/ghost/GHOST/download_scripts/GHOST_standards.py')



def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    urls = ['https://aqs.epa.gov/aqsweb/airdata/aqs_monitors.zip', 'https://aqs.epa.gov/aqsweb/airdata/aqs_sites.zip']
    filenames = ['aqs_monitors.zip', 'aqs_sites.zip']
    today = date.today()
    identifier = meta_params['identifier']

    os.makedirs('{}/US_EPA_AQS/metadata/{}/network_provided/temp/'.format(dl_root, version), exist_ok=True)
    download_location = '/{}/US_EPA_AQS/metadata/{}/network_provided/temp/'.format(dl_root, version)

    i=0
    for url in urls:
        
        n_tries = 0
        errcode = 999
        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(url, timeout=max_time_per_dl)
            if r.status_code == 200:
                urllib.request.urlretrieve(url, download_location+filenames[i])
                print('Downloaded {}'.format(filenames[i]))

                # unzip
                with zipfile.ZipFile(download_location+filenames[i], 'r') as zip_ref:
                    zip_ref.extractall(download_location)
        
                os.remove(download_location+filenames[i])
                errcode = r.status_code

            elif r.status_code == 404:
                print("No data found, error 404")
                errcode = 200

            elif r.status_code == 403:
                print("Permission denied for {}".format(url))
                errcode = 200

            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                max_time_per_dl = max_time_per_dl*2
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))


        # split downloaded file in csv files by state
        os.makedirs('{}/US_EPA_AQS/metadata/{}/network_provided/{}_{}/'.format(dl_root, version, filenames[i][4:-4], today.strftime('%Y%m%d')), exist_ok=True)
        os.makedirs('{}/US_EPA_AQS/metadata/{}/processed/{}/'.format(dl_root, version, filenames[i][4:-4]), exist_ok=True)

        with open('{}/US_EPA_AQS/metadata/{}/network_provided/temp/{}.csv'.format(dl_root, version, filenames[i][:-4]), 'r') as file:
            csv_filedata = csv.DictReader(file, delimiter=',')

            state_codes = []
            [state_codes.append(row['State Code']) for row in csv_filedata if row['State Code'] not in state_codes]
        
            for state_code in state_codes:
                print(state_code)
                file.seek(0)
                with open("{}/US_EPA_AQS/metadata/{}/network_provided/{}_{}/US_EPA_AQS_{}_{}_META_{}.csv".format(dl_root, version, filenames[i][4:-4], today.strftime('%Y%m%d'), filenames[i][4:-4], state_code, today.strftime('%Y%m%d')), "w") as f_out:
                    writer = csv.DictWriter(f_out, fieldnames=csv_filedata.fieldnames, delimiter=",")
                    writer.writeheader()
                    for row in csv_filedata:
                        if row['State Code'] == state_code:
                            writer.writerow(row)

        # JSON, process per state  ==========================================================================================================
        network = os.path.basename(__file__).split('_downloa')[0]
        present_meta_file_id = today.strftime('%Y%m%d')

        for state_code in state_codes:

            if meta_basefile == True:  # create json from base metadata file
                network_provided_meta_file = '{}/{}/metadata/{}/network_provided/{}_{}/{}_{}_{}_META_{}.csv'.format(dl_root, network, version, filenames[i][4:-4], meta_params['base_file_id'], network, filenames[i][4:-4], state_code, meta_params['base_file_id'])
                json_base_file = metadata_parsing_functions.create_json_file(dl_root, network, version, network_provided_meta_file, meta_params)

                # safe base file
                with open('{}/{}/metadata/{}/processed/{}/{}_{}_{}_META.json'.format(dl_root, network, version, filenames[i][4:-4], network, filenames[i][4:-4], state_code), 'w', encoding=meta_params['encoding']) as f:
                    f.write(json.dumps(json_base_file, indent=4))

            else:
                pass

            if meta_history == True: # parse all ever downloaded meta files
                print('Meta history parsing not implemented yet')
                pass
            #     # list all meta files
            #     meta_dir = '{}/{}/metadata/{}/network_provided/'.format(dl_root, network, version)
            #     network_provided_meta_files = [meta_dir+f for f in os.listdir(meta_dir) if (('META_' in f) and ('.csv' in f))]
            #     network_provided_meta_files = sorted(network_provided_meta_files)  # sort in right order
                
            #     for network_provided_meta_file in network_provided_meta_files: # loop through all files
            #         json_metadata_now = create_json_file(dl_root, network, version, network_provided_meta_file, meta_params)

            #         # compare current file with json basefile and update it if necessary
            #         compare_jsons(dl_root, network, version, json_metadata_now, meta_params)


            else: # only parse latest downloaded meta file
                # create json from current metadata file
                network_provided_present_meta_file = '{}/{}/metadata/{}/network_provided/{}_{}/{}_{}_{}_META_{}.csv'.format(dl_root, network, version, filenames[i][4:-4], present_meta_file_id, network, filenames[i][4:-4], state_code, present_meta_file_id)
                json_metadata_now = metadata_parsing_functions.create_json_file(dl_root, network, version, network_provided_present_meta_file, meta_params)

                # compare current file with json basefile and update it if necessary
                # modify 'network' variable to contain information about state and monitors/state, to find correct basefile in comparing-json-function
                network = network+'-'+filenames[i][4:-4]+'-'+state_code
                print(network)
                metadata_parsing_functions.compare_jsons(dl_root, network, version, json_metadata_now, meta_params)
                network = network.split('-')[0]


        i=i+1