"""
run the download of GHOST network data and metadata
"""

# import python modules
import time
import datetime
import traceback
import numpy as np
import os
import multiprocessing
import itertools
import importlib
import subprocess
import download_configuration as dlconfig


def download_network(network, dl_data, dl_metadata, download_params, version, mode):

    dl_start_time = time.time()
    max_time_per_dl = download_params[network]['max_time_dl']
    n_max_tries = download_params[network]['max_number_tries']
    dl_module = importlib.import_module(network+'_download') # import python script


    if dl_data == True:
        print("Started downloading data of {}".format(network))
        try:
            dl_module.download_data(mode, version, n_max_tries, max_time_per_dl)
            dl_mid_time = time.time()

            # copy data to gpfs
            if mode == "nrt":
                version = mode
            else:
                version = version

            subprocess.run(['./rsync_data.sh', network, version])

            dl_end_time = time.time()

            dl_time = np.round((dl_mid_time - dl_start_time)/60., 3)
            rsync_time = np.round((dl_end_time - dl_mid_time)/60., 3)
            total_dl_time = np.round((dl_end_time - dl_start_time)/60., 3)

            return '{} successful -- total: {} min +++ download: {} min +++ rsync: {} min \n-------------------------------------------------------------------------------------------------------------\n'.format(network, total_dl_time, dl_time, rsync_time)

        except Exception as e:
            dl_end_time = time.time()
            total_dl_time = np.round((dl_end_time - dl_start_time)/60., 3)
            print(e)
            return '{} unsuccessful -- total: {} min +++ download: {} min +++ rsync: {} min \n'.format(network, total_dl_time, dl_time, rsync_time)+str(traceback.format_exc())+'\n-------------------------------------------------------------------------------------------------------------\n'


    if dl_metadata == True:
        print("Started downloading metadata of {}".format(network))
        try:
            dl_module.download_metadata(n_max_tries, max_time_per_dl, version)

            dl_end_time = time.time()
            total_dl_time = np.round((dl_end_time - dl_start_time)/60., 3)
            return '{} successful -- duration: {} minutes \n-------------------------------------------------------------------------------------------------------------\n'.format(network, total_dl_time)
        
        except Exception as e:
            dl_end_time = time.time()
            total_dl_time = np.round((dl_end_time - dl_start_time)/60., 3)
            print(e)
            return '{} unsuccessful -- duration {} minutes \n'.format(network, total_dl_time)+str(traceback.format_exc())+'\n-------------------------------------------------------------------------------------------------------------\n'




if __name__ == "__main__":

    # import download configuration
    mode = dlconfig.mode
    version = dlconfig.version
    download_params = dlconfig.download_params
    dl_data = dlconfig.dl_data
    dl_metadata = dlconfig.dl_metadata
    networks = dlconfig.networks_to_download

    if dlconfig.networks_to_download == 'all':
        if mode == 'all':
            networks = dlconfig.all_networks
            
        elif mode == 'nrt':
            networks = dlconfig.all_nrt_networks


    print("Ghost version: {}".format(version))
    print("Mode: {}".format(mode))

    # create logfile
    dl_logfile = open('/esarchive/scratch/rgrodofz/software/ghost/GHOST/download_scripts/log_files/log_{}.out'.format(datetime.datetime.now().strftime('%Y%m%d-%H%M%S')),"w")
    dl_logfile.write('**************************************************************************************************************\n')        
    dl_logfile.write('BEGINNING DOWNLOADING DATA {}, GHOST version {}\n'.format(mode, version))
    dl_logfile.write('**************************************************************************************************************\n')

    # start downloading 
    dl_start_time = time.time()
    pool = multiprocessing.Pool(dlconfig.num_cpu)
    result = pool.starmap(download_network, zip(networks, itertools.repeat(dl_data), itertools.repeat(dl_metadata), itertools.repeat(download_params), itertools.repeat(version), itertools.repeat(mode)))
    dl_logfile.write(''.join(result))

    pool.close()
    pool.join()

    dl_end_time = time.time()
    dl_time_tot = np.round((dl_end_time - dl_start_time)/60., 3)
    dl_logfile.write('Total duration: {} minutes'.format(dl_time_tot))
    dl_logfile.close()


    # run the desired networks
    # for network in networks:

    #     dl_start_time = time.time()
    #     max_time_per_dl = download_params[network]['max_time_dl']
    #     n_max_tries = download_params[network]['max_number_tries']
    #     dl_module = importlib.import_module(network+'_download') # import python script

    #     if dl_data == True:
    #         print("Started downloading data of {}".format(network))
    #         try:
    #             dl_module.download_data(mode, version, n_max_tries, max_time_per_dl)

    #             # copy data to gpfs
    #             #os.system('rsync -arvzh /esarchive/obs/ghost/{}/ /gpfs/projects/bsc32/AC_cache/obs/ghost/{} > rsync_gpfs.txt'.format(network_name, network_name))

    #             dl_end_time = time.time()
    #             total_dl_time = (dl_end_time - dl_start_time)/60.
    #             #dl_logfile.write('{} successful -- duration: {} minutes'.format(network_name, total_dl_time))
    #             #dl_logfile.write('\n---------------------------------------------------------------------------------------------------\n')
                

    #         except Exception as e:
    #             print(e)
    #             dl_end_time = time.time()
    #             total_dl_time = np.round((dl_end_time - dl_start_time)/60., 3)
    #             print("failed {}".format(total_dl_time))
    #             #dl_logfile.write('{} unsuccessful -- duration {} minutes \n'.format(network_name, total_dl_time)+str(traceback.format_exc()))
    #             #dl_logfile.write('\n---------------------------------------------------------------------------------------------------\n')


    #     if dl_metadata == True:
    #         print("Started downloading metadata of {}".format(network))
    #         dl_module.download_metadata(n_max_tries, max_time_per_dl)

