import os
import sys
import requests
import logging
import urllib
import datetime as dt
import json
import csv
import pandas as pd
from pathlib import Path
from datetime import date
from datetime import timedelta
#from convert_json_csv import convert_json_csv


class aqs_epa_api(object):
    """defines and configures API to query the aqms database"""

    def __init__(self, ):
        self.logger = logging.getLogger(__file__)
        self.url_api = "https://aqs.epa.gov/data/api/"
        self.headers = {'content-type': 'application/json', 'accept': 'application/json'}

        self.get_states = 'list/states'
        self.get_counties = 'list/countiesByState'
        self.get_sites = 'list/sitesByCounty'

        self.get_parameters = "list/parametersByClass"
        self.get_data_by_site = 'sampleData/bySite'

        return
    
    def get_parameters_in_list(self, ):
        """build a query to return data by state"""
        query = urllib.parse.urljoin(self.url_api, self.get_parameters)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'pc': 'ALL'}
        response = requests.get(url=query, data='', params=params)
        return response

    def states(self, ):
        """build a query to return states in list"""
        query = urllib.parse.urljoin(self.url_api, self.get_states)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34'}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return print('Response error')
    
    def counties(self, code):
        """build a query to return list of counties by state"""
        query = urllib.parse.urljoin(self.url_api, self.get_counties)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'state': str(code)}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return 'Response error'

    def sites(self, code, ccode):
        """build a query to return list of sites per county"""
        query = urllib.parse.urljoin(self.url_api, self.get_sites)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'state': str(code), 'county': str(ccode)}
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return 'Response error'


    def get_data_site(self, code, ccode, scode):
        """build a query to return data by site"""
        query = urllib.parse.urljoin(self.url_api, self.get_data_by_site)
        params = {'email': 'raphael.grodofzig@bsc.es', 'key': 'bluewolf34', 'param': '44201', 'bdate': '20200101', 'edate': '20201231', 'state': str(code), 'county': str(ccode), 'site': str(scode)} # ozone
        response = requests.get(url=query, data='', params=params)
        if response.status_code == 200:
            return response.text
        else:
            return 'Response error'
    


def scraper(mode, version):

    if mode == 'all':
        bdate = date(2022, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today() + timedelta(days = 365)

        os.makedirs('/esarchive/obs/ghost/US_EPA_AQS/original_files/{}/'.format(version), exist_ok=True)

    elif mode == 'nrt':
        version = mode
        bdate = date.today() 
        edate = date.today() + timedelta(days = 365)
        download_location = '/esarchive/obs/ghost/US_EPA_AQS/original_files/nrt/'

    else:
        print('time mode inapplicable')

    
    components = ['O3']



    for component in components:
        # find all states
        states = json.loads(aqs_epa_api().states())['Data']
        for state in states:
            code = state.get('code')
            state_name = state.get('value_represented')
            print(state_name)

            # folder per component
            os.makedirs('/esarchive/obs/ghost/US_EPA_AQS/original_files/{}/{}/{}/'.format(version, component, state_name), exist_ok=True)
            download_location = '/esarchive/obs/ghost/US_EPA_AQS/original_files/{}/{}/{}/'.format(version, component, state_name)

            # find all counties
            counties = json.loads(aqs_epa_api().counties(code))['Data']
            for county in counties:
                ccode = county.get('code')
                county_name = state.get('value_represented')

                # find all sites
                sites = json.loads(aqs_epa_api().sites(code, ccode))['Data']
                for site in sites:
                    
                    scode = site.get('code')
                    site_name = site.get('value_represented')

                    if ((scode is not None) and (site_name is not None)): # maybe too strict ??? 
                        data_per_site = aqs_epa_api().get_data_site(code, ccode, scode)

                        # check if data is available for that site
                        if json.loads(data_per_site)['Header'][0]['status'] == 'Success':
                            # save files
                            filename = '{}_{}_{}_{}.json'.format(code, ccode, scode, component)
                            with open(download_location+filename, 'w') as outfile:
                                outfile.write(data_per_site)
                        else:
                            continue





    """
    AQMS = aqs_epa_api()
    #Allparameters = AQMS.get_parameters_in_list()
    data_per_site = AQMS.get_data_site().text

    print(data_per_site)


    # save parameter details to text file
    with open(download_location+'AQS_EPA_ozone_test.json', 'w') as outfile:
        outfile.write(data_per_site)"""
    """
    for item in Allparameters:
        item = item.decode('ISO-8859-1')
        f.write(item)

    f.close()


    input_path = "/home/rgrodofz/Documents/GHOST/US_EPA_AQS/EPA_ozone_via_API.json"
    output_path = "/home/rgrodofz/Documents/GHOST/US_EPA_AQS/EPA_ozone_via_API.csv"
    convert_json_csv(input_path, output_path)"""
    