from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from bs4 import BeautifulSoup
from chromedriver_py import binary_path

import tarfile 
import requests
import time
import os.path
import os
import shutil
import re
import pandas as pd
import csv
import json
from datetime import date
import numpy as np
import glob
import metadata_parsing_functions


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):
    # metadata included in files

    baseurl = 'https://wdcpc.org/data/index.html'
    
    if mode == "all":
        os.makedirs('{}/WMO_WDCPC/original_files/{}/'.format(dl_root, version), exist_ok=True)
        download_location = "{}/WMO_WDCPC/original_files/{}/".format(dl_root, version)

    elif mode == "nrt":
        os.makedirs('{}/WMO_WDCPC/original_files/nrt/'.format(dl_root), exist_ok=True)
        download_location = "{}/WMO_WDCPC/original_files/nrt/".format(dl_root)

    
    # set up driver
    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    # open url
    driver.get(baseurl)
    time.sleep(2)

    html = driver.page_source
    soup = BeautifulSoup(html, features="html.parser")

    stations = soup.find_all("a", href=re.compile(r".html"))

    for station in stations[1:]: # skip first link that is not a station
        link = station.get('href')
        url = 'https://wdcpc.org/data/'+link

        driver.get(url)
        time.sleep(3)

        html = driver.page_source
        soup = BeautifulSoup(html, features="html.parser")
        data_files = soup.find_all("a", href=re.compile(r".csv"))

        for data_file in data_files:
            file_name = data_file.get('href')
            print(file_name)
            
            #download
            n_tries = 0
            errcode = 999
            while (n_tries < n_max_tries) and (errcode != 200):
                r = requests.get('https://wdcpc.org/data/'+file_name, timeout=max_time_per_dl)
                if r.status_code == 200:
                    with open(download_location+'temp_'+file_name[4:], 'wb') as outfile:
                        outfile.write(r.content)
                    print('Downloaded {}'.format(file_name))
                    errcode = r.status_code

                elif r.status_code == 404:
                    print('No {}'.format(file_name))
                    errcode = 200 #skip
                    continue
                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    time.sleep(n_tries * 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))

            # check if data is given in two lines format and fix it before proceeding
            if 'HPB_20170101_20180101.csv' not in file_name: # erronous file, fix later???
                if 'HPB_20180101_20191231.csv' not in file_name:
                    if 'KWZ_20220104_20230103.csv' not in file_name:
                        if 'HPB_20060101_20090101.csv' not in file_name:

                            raw_file = pd.read_csv(download_location+'temp_'+file_name[4:], header=None, names=np.arange(0,64))
                            try:
                                header_index = raw_file.index[raw_file[0]=='GAW_ID'].tolist()[-1]
                            except:
                                header_index = raw_file.index[raw_file[0]=='GAW_ID'].tolist()

                            # METADATA, safe header
                            meta_data = raw_file.iloc[0:header_index,:]
                            meta_data.to_csv('{}/WMO_WDCPC/metadata/{}/network_provided/{}_META.csv'.format(dl_root, version, file_name[4:-4]), index=False, header=False)

                            # read DATA again with header
                            raw_file = pd.read_csv(download_location+'temp_'+file_name[4:], header=header_index)
                            #print(raw_file)

                            # fix GAW ID column
                            raw_file['GAW_ID'] = file_name[4:].split('_')[0]

                            # remove random "
                            for i in range(len(raw_file.iloc[0,:])):
                                for j in range(len(raw_file.iloc[:,0])):
                                    raw_file.iloc[j,i] = str(raw_file.iloc[j,i]).replace('"', '')

                            # delete double lines when start_date is none
                            valid_data = raw_file[raw_file['Start_Date'].notna()]

                            # safe
                            valid_data.to_csv('{}/WMO_WDCPC/original_files/{}/{}'.format(dl_root, version, file_name[4:]), index=False)
                            
            # remove temp file
            os.remove(download_location+'temp_'+file_name[4:])



            """
            # cut files in data and metadata
            station_ID = file_name.split('/')[1].split('_')[0]
            print(station_ID)

            if 'HPB_20060101_20090101.csv' not in file_name: # erronous file, fix later???
                if 'HPB_20180101_20191231.csv' not in file_name:
                    if 'KWZ_20220104_20230103.csv' not in file_name:
                        i=0
                        with open('{}/WMO_WDCPC/original_files/{}/{}'.format(dl_root, version, 'temp_'+file_name[4:]), 'r') as file:
                            csv_filedata = csv.DictReader(file, fieldnames=['GAW_ID','Start_Date','Start_Time','End_Date','End_Time','Time_Zone','Period_Flag','Precipitation_Type','Sample_Quality','Lab_Comment','Unusual_Occurrences','Guage_Depth','Guage_Flag','Sample_Mass','Mass_Flag','pH','pH_Flag','pH_DL_Flag','Cond','Cond_Flag','Cond_DL_Flag','Acidity','Acidity_Flag','Acidity_DL_Flag','SO4','SO4_Flag','SO4_DL_Flag','NH4','NH4_Flag','NH4_DL_Flag','NO3','NO3_Flag','NO3_DL_Flag','Cl','Cl_Flag','Cl_DL_Flag','Ca','Ca_Flag','Ca_DL_Flag','K','K_Flag','K_DL_Flag','Mg','Mg_Flag','Mg_DL_Flag','Na','Na_Flag','Na_DL_Flag','F','F_Flag','F_DL_Flag','PO4','PO4_Flag','PO4_DL_Flag','Zn','Zn_Flag','Zn_DL_Flag','Alcalinity','Alcalinity_Flag','Alcalinity_DL_Flag','IPD_FlagC','CPD_FlagC','CE_Flag','WDCPC_Comment']) #, delimiter=",")header=None)
                            #file.seek(0)

                            # DATA
                            with open('{}/WMO_WDCPC/original_files/{}/{}'.format(dl_root, version, file_name[4:]), "w") as f_out:
                                writer = csv.DictWriter(f_out, fieldnames=['GAW_ID','Start_Date','Start_Time','End_Date','End_Time','Time_Zone','Period_Flag','Precipitation_Type','Sample_Quality','Lab_Comment','Unusual_Occurrences','Guage_Depth','Guage_Flag','Sample_Mass','Mass_Flag','pH','pH_Flag','pH_DL_Flag','Cond','Cond_Flag','Cond_DL_Flag','Acidity','Acidity_Flag','Acidity_DL_Flag','SO4','SO4_Flag','SO4_DL_Flag','NH4','NH4_Flag','NH4_DL_Flag','NO3','NO3_Flag','NO3_DL_Flag','Cl','Cl_Flag','Cl_DL_Flag','Ca','Ca_Flag','Ca_DL_Flag','K','K_Flag','K_DL_Flag','Mg','Mg_Flag','Mg_DL_Flag','Na','Na_Flag','Na_DL_Flag','F','F_Flag','F_DL_Flag','PO4','PO4_Flag','PO4_DL_Flag','Zn','Zn_Flag','Zn_DL_Flag','Alcalinity','Alcalinity_Flag','Alcalinity_DL_Flag','IPD_FlagC','CPD_FlagC','CE_Flag','WDCPC_Comment']) #, delimiter=",")
                                writer.writeheader()

                                # META DATA
                                with open('{}/WMO_WDCPC/metadata/{}/network_provided/{}_META.csv'.format(dl_root, version, file_name[4:-4]), "w") as f_meta:
                                    writer_meta = csv.DictWriter(f_meta, fieldnames=csv_filedata.fieldnames) #, fieldnames=['GAW_ID','Start_Date','Start_Time','End_Date','End_Time','Time_Zone','Period_Flag','Precipitation_Type','Sample_Quality','Lab_Comment','Unusual_Occurrences','Guage_Depth','Guage_Flag','Sample_Mass','Mass_Flag','pH','pH_Flag','pH_DL_Flag','Cond','Cond_Flag','Cond_DL_Flag','Acidity','Acidity_Flag','Acidity_DL_Flag','SO4','SO4_Flag','SO4_DL_Flag','NH4','NH4_Flag','NH4_DL_Flag','NO3','NO3_Flag','NO3_DL_Flag','Cl','Cl_Flag','Cl_DL_Flag','Ca','Ca_Flag','Ca_DL_Flag','K','K_Flag','K_DL_Flag','Mg','Mg_Flag','Mg_DL_Flag','Na','Na_Flag','Na_DL_Flag','F','F_Flag','F_DL_Flag','PO4','PO4_Flag','PO4_DL_Flag','Zn','Zn_Flag','Zn_DL_Flag','Alcalinity','Alcalinity_Flag','Alcalinity_DL_Flag','IPD_FlagC','CPD_FlagC','CE_Flag','WDCPC_Comment']) #, delimiter=",")
                                    #writer_meta.writeheader()

                                    for row in csv_filedata:
                                        if row['GAW_ID'] == station_ID:
                                            if i==0:
                                                writer_meta.writerow(row)
                                                i=1
                                            else: # DATA
                                                writer.writerow(row)
                                        else:
                                            writer_meta.writerow(row)

            # remove temp file
            os.remove(download_location+'temp_'+file_name[4:])


    # # check if files are empty and delete empty files
    # dl_files = glob.glob(download_location+'*')
    # print(dl_files)
    # for dl_file in dl_files:
    #     data = pd.read_csv(dl_file, header=0)
    #     if data.empty:
    #         os.remove(dl_file)"""




def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):
    # metadata is downloaded already with data files, only needs formatting to json
    # json parsing varies from standard json parsing
    
    today = date.today().strftime('%Y%m%d')
    network = os.path.basename(__file__).split('_downloa')[0]
    json_metadata = {}
    max_num_files_per_station = 20

    all_network_provided_files = os.listdir('{}/{}/metadata/{}/network_provided/'.format(dl_root, network, version))

    # get station IDs
    station_IDs = []
    for file in all_network_provided_files:
        station_IDs.append(file.split('_')[0])
    station_IDs = list(dict.fromkeys(station_IDs)) # drop duplicates

    # get files for station
    station_file_dict = {}
    for station_ID in station_IDs:
        station_file_dict[station_ID] = []
        for file in all_network_provided_files:
            if station_ID in file:
                station_file_dict[station_ID].append(file)
        
        # sort in time
        station_file_dict[station_ID] = sorted(station_file_dict[station_ID])
    
    # BASE FILE
    for station_ID in station_file_dict:

        json_metadata[station_ID] = {}
        station_file = station_file_dict[station_ID][0] # first file as base file!
        json_basefile = parse_meta_info_per_file(dl_root, network, version, station_ID, station_file, meta_params, json_metadata)

    with open('{}/{}/metadata/{}/processed/{}_META.json'.format(dl_root, network, version, network), 'w', encoding=meta_params['encoding']) as f:
        f.write(json.dumps(json_basefile, indent=4))

    # OTHER FILES
    json_metadata = {}

    # other files, needs extra loop
    # loop through second, third, etc files per station ID
    for i in range(1, max_num_files_per_station): # leave out first file as it was used for base file
        for station_ID in station_file_dict:
            json_metadata[station_ID] = {}
            if len(station_file_dict[station_ID]) >= i+1: # check if there is second, third, etc file
                station_file = station_file_dict[station_ID][i]
                json_file = parse_meta_info_per_file(dl_root, network, version, station_ID, station_file, meta_params, json_metadata)

        metadata_parsing_functions.compare_jsons(dl_root, network, version, json_file, meta_params)



def parse_meta_info_per_file(dl_root, network, version, station_ID, station, meta_params, json_metadata):

    update_date_raw = station.split('_')[1]
    update_date = update_date_raw[:4]+'-'+update_date_raw[4:6]+'-'+update_date_raw[-2:]

    identifier = station_ID
    delimiter_csv = meta_params['delimiter_meta_csv']

    file = pd.read_csv('{}/{}/metadata/{}/network_provided/{}'.format(dl_root, network, version, station), header=None, keep_default_na=False)

    # locating and extracting info from pandas frame
    lat_row, lat_col = np.where(file == 'Latitude')
    lat = file.iloc[lat_row[0]+1, lat_col[0]]

    lon_row, lon_col = np.where(file == 'Longitude')
    lon = file.iloc[lon_row[0]+1, lon_col[0]]

    ele_row, ele_col = np.where(file == 'Elevation')
    alt = file.iloc[ele_row[0]+1, ele_col[0]]

    lab_row, lab_col = np.where(file == 'Laboratory_ID')
    lab_ID = file.iloc[lab_row[0]+1, lab_col[0]]

    samp_period_row, samp_period_col = np.where(file == 'Sampling_Period')
    sampling_period = file.iloc[samp_period_row[0]+1, samp_period_col[0]]

    samp_type_row, samp_type_col = np.where(file == 'Sampling_Type')
    sampling_type = file.iloc[samp_type_row[0]+1, samp_type_col[0]]

    samp_pres_row, samp_pres_col = np.where(file == 'Sampling_Preservation')
    sampling_preservation = str(file.iloc[samp_pres_row[0]+1, samp_pres_col[0]]) + ' ' + str(file.iloc[samp_pres_row[0]+1, samp_pres_col[0]+1])

    meas_row, meas_col = np.where(file == 'Measurement')
    gaw_id_row, gaw_id_col = np.where(file == 'GAW_ID')
    measurements = file.iloc[meas_row[0]+1:, meas_col[0]:meas_col[0]+4]

    measurement_methods = pd.DataFrame(measurements)

    # unify column names
    component_names = {'SO4': 'Sulfate', 'NO3': 'Nitrate', 'NH4': 'Ammonium', 'F': 'Fluoride', 'Cl': 'Chloride', 'Na': 'Sodium', 'K': 'Potassium', 'Ca': 'Calcium', 'Mg': 'Magnesium'}
    for i in range(len(measurement_methods)): # loop through rows
        for component_name in component_names:
            if component_names[component_name] in measurement_methods.iloc[i,0]:
                measurement_methods.iloc[i,0] = component_name


    # parse json
    json_metadata[station_ID]['lab_ID'] = {'values': [lab_ID], 'update_time': [update_date]}
    json_metadata[station_ID]['Latitude'] = {'values': [lat], 'update_time': [update_date]}   
    json_metadata[station_ID]['Longitude'] = {'values': [lon], 'update_time': [update_date]}   
    json_metadata[station_ID]['Altitude'] = {'values': [alt], 'update_time': [update_date]}
    json_metadata[station_ID]['Sampling_Period'] = {'values': [sampling_period], 'update_time': [update_date]}  
    json_metadata[station_ID]['Sampling_Type'] = {'values': [sampling_type], 'update_time': [update_date]}  
    json_metadata[station_ID]['Sampling_Preservation'] = {'values': [sampling_preservation], 'update_time': [update_date]}
    json_metadata[station_ID]['Guage_Depth_method'] = {'values': ['rain gauge'], 'update_time': [update_date]}
    json_metadata[station_ID]['Guage_Depth_detection_limit'] = {'values': ['0'], 'update_time': [update_date]}
    json_metadata[station_ID]['Guage_Depth_units'] = {'values': ['mm'], 'update_time': [update_date]}

    
    for i in range(len(measurement_methods)):
        json_metadata[station_ID][str(measurement_methods.iloc[i,0])+'_method'] = {'values': [measurement_methods.iloc[i,1]], 'update_time': [update_date]}
        json_metadata[station_ID][str(measurement_methods.iloc[i,0])+'_detection_limit'] = {'values': [measurement_methods.iloc[i,2]], 'update_time': [update_date]}
        json_metadata[station_ID][str(measurement_methods.iloc[i,0])+'_units'] = {'values': [measurement_methods.iloc[i,3]], 'update_time': [update_date]}


    # check for punctuation of lat, lon, alt, height
    for punct_variable in ['Latitude', 'Longitude', 'Altitude', 'Height']: # standard GHOST names
        if punct_variable in meta_params.keys():
            for network_spec_var_name in meta_params[punct_variable]: # loop through all possible variable names in one network, usually it should just be one name 
                for station in json_metadata: # loop through all stations
                    if network_spec_var_name in json_metadata[station]: # clean punctuation discrepancies
                        input_val = json_metadata[station][network_spec_var_name]['values'][0] # check values
                        if input_val is not None:
                            input_val = input_val.replace(',', '.').replace(' ', '').replace('E', '').replace('N', '')

                            if input_val == '': # check if its empty string
                                output_val = None
                            elif input_val == '-':  # check if its dash
                                output_val = None
                            else:
                                try:
                                    output_val = str(float(input_val))
                                except: # if not convertible to float, disregard
                                    output_val = None

                        json_metadata[station][network_spec_var_name]['values'][0] = output_val
                        
                    else:
                        continue
        else:
            continue

    return json_metadata