import os
import sys
import requests
import logging
import urllib
import datetime as dt
import json
import csv
import pandas as pd
from pathlib import Path
from datetime import date
from datetime import timedelta
from GHOST_standards import standard_parameters
#from convert_json_csv import convert_json_csv


class aqs_epa_api(object):
    """defines and configures API to query the aqms database"""

    def __init__(self, ):
        self.logger = logging.getLogger(__file__)
        self.url_api = "https://aqs.epa.gov/data/api/"
        self.headers = {'content-type': 'application/json', 'accept': 'application/json'}

        self.get_states = 'list/states'
        self.get_counties = 'list/countiesByState'
        self.get_sites = 'list/sitesByCounty'

        self.get_parameters = "list/parametersByClass"
        self.get_data_by_site = 'sampleData/bySite'
        self.get_Dailydata_by_site = 'dailyData/bySite'

        return
    
    def get_parameters_in_list(self, ):
        """build a query to return data by state"""
        query = urllib.parse.urljoin(self.url_api, self.get_parameters)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'pc': 'ALL'}
        response = requests.get(url=query, data='', params=params)
        return response

    def states(self, ):
        """build a query to return states in list"""
        query = urllib.parse.urljoin(self.url_api, self.get_states)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34'}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return print('Response error')
    
    def counties(self, code):
        """build a query to return list of counties by state"""
        query = urllib.parse.urljoin(self.url_api, self.get_counties)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'state': str(code)}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return 'Response error'

    def sites(self, code, ccode):
        """build a query to return list of sites per county"""
        query = urllib.parse.urljoin(self.url_api, self.get_sites)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'state': str(code), 'county': str(ccode)}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return 'Response error'

    def request_data(self, n_max_tries, max_time_per_dl, query, params):
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            response = requests.get(url=query, data='', params=params, timeout=max_time_per_dl)
                    
            if response.status_code == 200:
                return response.text
                errcode = response.status_code
                
            else:
                # try again
                print('Response error {}, attempt {}'.format(response.status_code, n_tries))
                errcode = response.status_code
                n_tries += 1

        if n_tries == n_max_tries:
            print('Failed downloading {}_{}_{}_{}_{} {} times in {} seconds, error code {}'.format(code, ccode, scode, pcode, bbdate[:4], n_tries, max_time_per_dl, errcode))
            return None


    def get_data_site(self, code, ccode, scode, pcode, bbdate, eedate):
        """build a query to return data by site"""
        query = urllib.parse.urljoin(self.url_api, self.get_data_by_site)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'param': pcode, 'bdate': bbdate, 'edate': eedate, 'state': str(code), 'county': str(ccode), 'site': str(scode)}
        return query, params


    def get_Dailydata_site(self, code, ccode, scode, pcode, bbdate, eedate):
        """build a query to return data by site"""
        query = urllib.parse.urljoin(self.url_api, self.get_Dailydata_by_site)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'param': pcode, 'bdate': bbdate, 'edate': eedate, 'state': str(code), 'county': str(ccode), 'site': str(scode)}
        return query, params

    


def download_data(mode, version, n_max_tries, max_time_per_dl):

    if mode == 'all':
        bdate = date(2022, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)

        os.makedirs('/esarchive/obs/ghost/US_EPA_AQS/original_files/{}/'.format(version), exist_ok=True)

    elif mode == 'nrt':
        version = mode
        bdate = date.today() - timedelta(days = 1)
        edate = date.today() 
        download_location = '/esarchive/obs/ghost/US_EPA_AQS/original_files/nrt/'

    else:
        print('time mode inapplicable')


    time_resolutions = ['hourly', 'daily']
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()
    print(years)

    components = {}
    for standard_parameter in standard_parameters:
        aqs_code = standard_parameters[standard_parameter]['aqs_code']
        if aqs_code: # check if aqs_code is available
            components[standard_parameter] = standard_parameters[standard_parameter]['aqs_code']


    if mode == 'all': # use US EPA AQS API

        # find all states
        states = json.loads(aqs_epa_api().states())['Data']
        for state in states:
            code = state.get('code')
            state_name = state.get('value_represented')
            print('Downloading... '+state_name)

            # find all counties
            counties = json.loads(aqs_epa_api().counties(code))['Data']
            for county in counties:
                ccode = county.get('code')
                county_name = state.get('value_represented')

                # find all sites
                sites = json.loads(aqs_epa_api().sites(code, ccode))['Data']
                for site in sites:
                    scode = site.get('code')
                    site_name = site.get('value_represented')

                    if ((scode is not None) and (site_name is not None)): # maybe too strict ???

                        for time_resolution in time_resolutions:

                            for component in components:
                                pcode = components[component][0] # select first aqs_code if there are multiple

                                for year in years:
                                    # yearly files
                                    bbdate = date(int(year), 1, 1).strftime('%Y%m%d')
                                    eedate = date(int(year), 12, 31).strftime('%Y%m%d')

                                    if time_resolution == 'hourly':
                                        query, params = aqs_epa_api().get_data_site(code, ccode, scode, pcode, bbdate, eedate)
                                    elif time_resolution == 'daily':
                                        query, params = aqs_epa_api().get_Dailydata_site(code, ccode, scode, pcode, bbdate, eedate)
                                    else:
                                        print('Time resolution problem')

                                    data_per_site = aqs_epa_api().request_data(n_max_tries, max_time_per_dl, query, params)
                                    if data_per_site == None:
                                        continue # skip if download failed

                                    # check if data is available for that site
                                    if json.loads(data_per_site)['Header'][0]['status'] == 'Success':
                                        # save files
                                        filename = '{}_{}_{}_{}_{}_{}.json'.format(time_resolution, code, ccode, scode, pcode, year)

                                        # folder per year
                                        os.makedirs('/esarchive/obs/ghost/US_EPA_AQS/original_files/{}/{}/{}/'.format(version, component, year), exist_ok=True)
                                        download_location = '/esarchive/obs/ghost/US_EPA_AQS/original_files/{}/{}/{}/'.format(version, component, year)

                                        with open(download_location+filename, 'w') as outfile:
                                            outfile.write(data_per_site)
                                        print("Downloaded {}".format(filename))
                                    else:
                                        print('No data found for {}_{}_{}_{}_{}_{}'.format(time_resolution, code, ccode, scode, pcode, year))
                                        continue


    elif mode == 'nrt': # use AirNow

        base_url = 'https://s3-us-west-1.amazonaws.com//files.airnowtech.org/airnow/{}/{}/HourlyData_{}.dat'

        # create hourly array
        hours = pd.date_range(bdate, edate, freq='H').strftime('%Y%m%d%H').tolist()[:-1]
        print(hours)

        # download

        for hour in hours:
            
            url = base_url.format(hour[:4], hour[:8], hour)
            r = requests.get(url, timeout=60)
            if r.status_code == 200:
                urllib.request.urlretrieve(url, download_location+'HourlyData_'+hour+'.dat')
                print('Downloaded {}'.format(url))
            else:
                print('No {}'.format(url))

    