import certifi
import socket
import subprocess
from urllib.error import HTTPError, URLError
from urllib.request import urlopen
import re
import os
from datetime import date
from datetime import timedelta
import requests
import csv
import json
import time
import metadata_parsing_functions


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    if mode == 'all':
            start_year = 1970
            end_year = 2024

    elif mode == 'nrt':
        start_year = start.strftime('%Y')
        end_year = (stop + timedelta(days=365)).strftime('%Y')
        version = mode

    else:
        print('time mode inapplicable')

    #iterate through years
    for year in range(int(start_year), int(end_year)):
        print(year)

        link_url = 'https://www.ncei.noaa.gov/data/global-hourly/archive/isd'

        #catch server connection exceptions
        read_url = False
        while read_url == False:
            try:
                link_data = re.findall("href=[\"\'](.*?)[\"\']", urlopen(link_url, timeout=max_time_per_dl, cafile=certifi.where()).read().decode('utf-8-sig'))
                read_url = True
            except HTTPError as error:
                print('Data not retrieved because %s\nURL: %s'%(error, link_url))
            except URLError as error:
                print('Data not retrieved because %s\nURL: %s'%(error, link_url))
            except socket.timeout:
                print('socket timed out - URL: %s'%(link_url))

        #keep only links which end in .csv
        link_list = ['{}/{}'.format(link_url,lnk) for lnk in link_data if 'isd_{}'.format(year) in lnk]   

        #dir to save files
        os.makedirs('{}/NOAA_ISD/original_files/{}/meteo/{}/'.format(dl_root, version, year), exist_ok=True)
        specific_directory = '{}/NOAA_ISD/original_files/{}/meteo/{}/'.format(dl_root, version, year)

        #iterates through each link and downloads to required directory
        #checks if remote file required already exists and needs updating
        #handles issue of server hanging for 3 minutes spoaradically

        #try downloading each link a certain number of times before giving up
        for link in link_list:
            n_tries = 0
            errcode = 999
            while (n_tries < n_max_tries) & (errcode != 0):
                if n_tries == 0:
                    print('Checking/Downloading %s'%(link))
                else:
                    print('*** Previous check/download failed. Re-trying for %s'%(link))
                cmd = 'wget -N -P %s %s -q -o /dev/null'%(specific_directory,link)
                process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', shell=True)
                status = process.communicate()[0]
                errcode = process.returncode
                if errcode != 0:
                    n_tries+=1

            #untar file
            lnk = link.split('/')[-1]
            cmd = 'tar -xf {}/{} -C {}'.format(specific_directory,lnk,specific_directory)
            print('Un-tarring file')
            process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', shell=True)
            status = process.communicate()[0]
            errcode = process.returncode

            #remove isd history 
            cmd = 'rm {}/isd-history*'.format(specific_directory)
            process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', shell=True)
            status = process.communicate()[0]
            errcode = process.returncode

            #remove tar file
            cmd = 'rm {}/{}'.format(specific_directory,lnk)
            process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', shell=True)
            status = process.communicate()[0]
            errcode = process.returncode

def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    url_metadata = 'https://www.ncei.noaa.gov/pub/data/noaa/isd-history.csv'
    download_location = dl_root+"/NOAA_ISD/metadata/{}/network_provided/NOAA_ISD_META_{}.csv"

    n_tries = 0
    errcode = 999
    today = date.today()
    
    while (n_tries < n_max_tries) and (errcode != 200):
        r = requests.get(url_metadata, timeout=max_time_per_dl)
        if r.status_code == 200:
            with open(download_location.format(version, today.strftime('%Y%m%d')), 'wb') as outfile:
                outfile.write(r.content)
            print('Downloaded metadata')
            errcode = r.status_code
        elif r.status_code == 404:
            print("No metadata found, error 404")
            errcode = 200
        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            max_time_per_dl = max_time_per_dl*2 # increase waiting time 
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_metadata, n_tries, max_time_per_dl, errcode))
    time.sleep(1)

    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)