###------------------------------------------------------------------------------------###
#*****NO LONGER USED*****

#converts data/metadata dictionary array to numpy record array, to allow indexing with column names

def convert_to_numpy_record_array(dictionary_to_convert):
    
    #get dictionary keys
    keys = dictionary_to_convert.keys()
    converted_array = [[] for _ in xrange(len(keys))]
    
    for ii,k in enumerate(keys):
        converted_array[ii].extend(dictionary_to_convert[k]['value']) 
                       
    #now convert to numpy record array
    converted_array = np.rec.fromarrays(converted_array, names=keys)
    
    return converted_array

#*****NO LONGER USED*****
    
###------------------------------------------------------------------------------------###

#*****NO LONGER USED*****

#deletes one or multiple fields from numpy dataframe/record array 

def delete_field_from_numpy_dataframe(array, *fieldnames_to_remove):    
    return array[ [ name for name in array.dtype.names if name not in fieldnames_to_remove ] ]
    
#*****NO LONGER USED*****

###------------------------------------------------------------------------------------###
      
#*****NO LONGER USED*****

#convert metadata/data read from each into consistent data formats
#each of the possible metadata/data fields have standardised data formats 

#for string types which contain np.NaN values for missing data --> np.NaN values are converted to the string 'nan' (always lower case)
#for numerical types which contain np.NaN values for missing data --> np.NaN values are preserved as the floating point np.NaN types 

#replace all commas with dashes "-" for strings, to prevent issues with comma delimiters later  

def standardise_data_formats(metadata,data):
    
    #get metadata/data keys
    metadata_keys = metadata.keys()
    data_keys = data.keys()
    
    metadata_format_dict = standardisation_modules.create_standardised_metadata_frame()
    data_format_dict     = standardisation_modules.create_standardised_data_frame()
    
    data_types = [metadata,data]
    data_keys = [metadata_keys,data_keys]
    format_dicts = [metadata_format_dict,data_format_dict]
        
    #format data/metadata
    for ii, data_type in enumerate(data_types):
        #iterate through each parameter field 
        for key in data_keys[ii]:           
            
            #get saved array for each parameter
            param_array = data_type[key]['value']
        
            #get defined data type for each parameter 
            param_data_type = format_dicts[ii][key]['data_type']
        
            #get defined string formatting for each parameter
            param_string_format = format_dicts[ii][key]['string_format']
        
            #get defined number of decimal places
            param_n_decimals = format_dicts[ii][key]['n_decimals']
        
            #do formatting
        
            #------------------------------------------
            #float types 
            #------------------------------------------
            if (param_data_type == np.float32) or (param_data_type == np.float64):
            
                #no rounding
                if pd.isnull(param_n_decimals) == True:
                    param_array = np.array(param_array, dtype=param_data_type)
                #rounding
                else:
                    param_array = np.around(np.array(param_array, dtype=param_data_type), param_n_decimals)
    
            #------------------------------------------
            #integer types 
            #NOTE - NaNs are floats by nature, therefore attempting to convert to integer will break processing.
            #however, fortunately all parameters with set integer types are impossible to contain NaNs.
            #these parameters for record are: time, wmo_region, meta_update_stamp, data_download_stamp
            #------------------------------------------
            elif (param_data_type == np.int32) or (param_data_type == np.int64) or (param_data_type == np.uint32) or (param_data_type == np.uint64):        
                
                param_array = np.array(param_array, dtype=param_data_type)
                #param_array = [param_data_type(x) if pd.isnull(x) == False else x for x in param_array]
                
            #------------------------------------------
            #string types
            #------------------------------------------
            elif (param_data_type == '|S%s'%(run_details['max_character_limit_short'])) or (param_data_type == '|S%s'%(run_details['max_character_limit_long'])):
                
                #upper case string formatting
                if param_string_format == 'upper':
                    param_array = [np.str(x).replace(",", "-").upper() if pd.isnull(x) == False else x for x in param_array]
        
                #lower case string formatting
                elif param_string_format == 'lower':
                    param_array = [np.str(x).replace(",", "-").lower() if pd.isnull(x) == False else x for x in param_array]
            
                #titalling string formatting
                elif param_string_format == 'title':                
                    param_array = [string.capwords(np.str(x)).replace(",", "-") if pd.isnull(x) == False else x for x in param_array]
        
                #convert to correct data type - object (is converted to correct string length when writing to NetCDF)
                param_array = np.array(param_array, dtype=object)
        
            #resave array
            data_types[ii][key]['value'] = param_array
        
    return metadata,data

#*****NO LONGER USED*****

###------------------------------------------------------------------------------------###

#*****NO LONGER USED*****

#read netcdf into numpy dataframe

def read_from_netcdf(netcdf_file):
    
    #read netcdf file
    netcdf_array = Dataset(netcdf_file) 
    
    #get keys in netcdf array
    netcdf_keys = netcdf_array.variables.keys()
    
    #create data dictionary frames to place metadata/data 
    metadata = standardisation_modules.create_standardised_metadata_frame()
    data =     standardisation_modules.create_standardised_data_frame()
    
    #place netcdf data into data dictionaries
    #convert character to object types
    for data_dict in [metadata,data]:
        for key_i, key in enumerate(netcdf_keys): 
            if data_dict[key]['data_type'] == np.dtype('O'):
                data_array[key_i]['value'] = chartostring(netcdf_array[key][:]).astype(np.object)
            else:   
                data_array[key_i]['value'] = netcdf_array[key][:]

    #convert data dictionaries to numpy data frame
    metadata = convert_to_numpy_dataframe(metadata)
    data = convert_to_numpy_dataframe(data)
    
    return metadata, data

#*****NO LONGER USED*****

###------------------------------------------------------------------------------------###

#*****NO LONGER USED*****

#module that handles processing network data in parallel safely
#this stems from issue where processing breaks from structerror for unknown reasons (believe it is error in python multiprocessing module)

#WHEN PROCESSING BREAKS ALL PROCESSES ARE TERMINATED, AND THE ALREADY PROCESSED STATIONS ARE DETERMINED (I.E. HAVE FILES IN TEMPORARY FILE DIRECTORY) AND ARE REMOVED FROM ITERERABLE LIST TO PREVENT PROCESSING THEM AGAIN 
#PROCESSING IS THEN RESUMED IN PARALLEL AND CONTINUES UNTIL ALL PROCESSING COMPLETES CORRECTLY

def run_parallel_safe(function, var_to_iterate, parallel_safe_dir):

    valid_process = False
    while valid_process == False:
        try:
            pool = multiprocessing.Pool(processes=int(run_details['process_type'].replace('PARALLEL','')))        
            pool.map(function,var_to_iterate)
            pool.close()
            pool.join()  
            valid_process = True  
        except (struct.error, IndexError), err:
            print '********************************************************************************************************************'
            print 'PARALLEL PROCESSING HAS FAILED. REMOVE STATION REFERENCES ALREADY PROCESSED FROM ZIPPED LIST AND RESTART PROCESSING'
            print '********************************************************************************************************************'
            pool.terminate()
            pool.join() 
            #QUICK SLEEP TO GIVE MUTIPROCESSING A MOMENT TO TERMINATE EVERYTHING
            time.sleep(10)
            #IF HAVE PROCESSED OUTPUT (I.E. FILES WITH FILENAMES OF STATION REFERENCES IN PARALLEL SAFE FILE DIRECTORY) 
            #THEN CUT ZIPPED LIST TO ENSURE THEY ARE NOT PROCESSED AGAIN
            
            #IF VARIBALE THAT YOU ARE ITERATING IS A ZIPPED LIST, PULL OUT JUST STATION REFERENCE
            #OTHERWISE, VARIABLE ITERATING WILL BE JUST BE STATION RERFERENCES
            #if len(np.array(var_to_iterate).shape) > 1:
            current_refs = zip(*var_to_iterate)[0]
            #else:
            #current_refs = var_to_iterate
            
            processed_refs = []
            processed_inds = []
            #get all currently processed references
            for file in glob.glob('%s*'%(parallel_safe_dir)):
                processed_refs.append(file.split('/')[-1])
            #get index of all processed references in current list of references (if exist in current list of references)
            for processed_ref in processed_refs:
                if processed_ref in current_refs:
                    processed_inds.append(current_refs.index(processed_ref))
            for remove_i in sorted(processed_inds, reverse=True):
                del var_to_iterate[remove_i]
            #CLEAN PARALLEL SAFE DIRECTORY IN CASE PARALLEL PROCESSING BREAKS AGAIN DURING THIS RUN (WOULD END UP TRYING TO REMOVE REFERENCE FROM ZIPPED LIST THAT YOU HAVE ALREADY REMOVED)
            #for file in glob.glob('%s*'%(parallel_safe_dir)):
            #    os.remove(file)
            #QUICK SLEEP JUST TO MAKE SURE TEMPORARY FILES HAVE BEEN REMOVED FROM SYSTEM BEFORE RESTARTING RUN
            time.sleep(10)
 
    return
    


###------------------------------------------------------------------------------------###

#*****NO LONGER USED*****

#module to run functions in parallel. Requires a function and iterable (can be zipped multivariable array)

def run_in_parallel(function, iterable):

    pool = multiprocessing.Pool(processes=int(run_details['process_type'].replace('PARALLEL','')))
    returned_data = pool.map(function, iterable)
    pool.close()
    pool.join()
    
    #if nothing returned from process, return nothing
    if all(v is None for v in returned_data) == True:
        return
    #otherwise return it
    else:
        return returned_data
        
#*****NO LONGER USED*****

###------------------------------------------------------------------------------------###
