#!/usr/bin/env python
# -*- coding: utf-8 -*-

#WRITTEN BY DENE BOWDALO

###------------------------------------------------------------------------------------###
###IMPORT MODULES
###------------------------------------------------------------------------------------###

import __builtin__
import crossnetwork_processing_modules
import datetime
import gc
import glob
import numpy as np
import os
import pandas as pd
import time

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###

#MODULE FOR GRIDDING/PROCESSING NETWORK/PARAMETER DATA FROM NATIVE RESOLUTION .npz STATION FILES INTO STANDARD NETCDF BSC CONVENTIONS FOR USE WITH ET TOOL 
#FOR EACH NETWORK/PARAMETER PAIRING, THE SEPARATE FILES FOR THE DIFFERENT STATION REFERENCES ARE MERGED TO CREATE MONTHLY NETCDF FILES BETWEEN START DATE AND END DATE

#INPUT OPTIONS:

#NETWORK (list of network processed files that you want to convert) 
#AIRBASE, AQS, CAPMON, CASTNET, EANET, EBAS, EIONET, GAW, NAPS, SEARCH

#PARAMETERS (list of parameters that you wish to process)
#Gases:                     O3, NO, NO2, CO, SO2, NH3, HNO3, ISOPRENE
#Aerosol - bulk:            PM10, PM2.5, PM1, C, Cl-, EC,  Na+, NH4+, NO3-, OC, SO4--, SO4--_NSS
#Aerosol - PM10 speciated:  PM10_C, PM10_C_corrected, PM10_Cl-, PM10_EC, PM10_Na+, PM10_NH4+, PM10_NO3-, PM10_OC, PM10_OC_corrected, PM10_SO4--, PM10_SO4--_NSS
#Aerosol - PM2.5 speciated: PM2.5_C, PM2.5_C_corrected, PM2.5_Cl-, PM2.5_EC, PM2.5_Na+, PM2.5_NH3, PM2.5_NH4+, PM2.5_NO3-, PM2.5_OC, PM2.5_OC_corrected, PM2.5_SO4--, PM2.5_SO4--_NSS
#Aerosol - PM1 speciated:   PM1_C, PM1_C_corrected, PM1_Cl-, PM1_EC, PM1_Na+, PM1_NH4+, PM1_NO3-, PM1_OC, PM1_SO4--, PM1_SO4--_NSS

#START_DATE = YYYYMM (start year/month for gridding)
#END_DATE = YYYYMM (end year/month of gridding)

#TEMPORAL_RESOLUTION = H, D, M (temporal resolution desired of merged files: hourly, daily or monthly)

#PROCESS_TYPE = SERIAL, PARALLELN (where N = number of processes to start)

#***All temporal resolutions will be converted first for one parameter, for one network***#

###------------------------------------------------------------------------------------###

networks = ['EBAS']
parameters = ['O3']
#parameters = ['O3','NO','NO2','CO','SO2','NH3','HNO3','ISOPRENE']
#parameters = ['PM10','PM2.5','PM1']
#parameters = ['C','Cl-','EC','Na+','NH4+','NO3-','OC','SO4--']
#parameters = ['PM10_C','PM10_Cl-','PM10_EC','PM10_Na+','PM10_NH4+','PM10_NO3-','PM10_OC','PM10_SO4--']
#parameters = ['PM2.5_C','PM2.5_Cl-','PM2.5_EC','PM2.5_Na+','PM2.5_NH4+','PM2.5_NO3-','PM2.5_OC','PM2.5_SO4--']
#parameters = ['PM1_C','PM1_Cl-','PM1_EC','PM1_Na+','PM1_NH4+','PM1_NO3-','PM1_OC','PM1_SO4--']
start_date = '197001'
end_date = '201801'
temporal_resolutions = ['H','D','M']
process_type = 'PARALLEL6'

#OPTION TO FILTER OUTPUT BY KEY QA CHECKS, SCREENING OUT BAD DATA, BEFORE OUTPUT TO NETCDF
filter_output = False
__builtin__.filter_output = filter_output

###------------------------------------------------------------------------------------###
if __name__ == '__main__':

    for network in networks:
        for parameter in parameters:  
        
            #GET SPECIFIC CORE PARAMETER DETAILS AND MAKE GLOBAL
            parameter_details = crossnetwork_processing_modules.get_parameter_details()[parameter]
            __builtin__.parameter_details = parameter_details
        
            for res in temporal_resolutions:
                #START TIMER
                t0 = time.time()
                
                #WHICH NETWORK, WHICH PARAMETER, WHICH TEMPORAL RESOLUTION?
                print network, parameter, res

                #GET CORE RUN DETAILS AND MAKE GLOBAL
                run_details = {'network':network,'parameter':parameter,'start_date':start_date,'end_date':end_date,'process_type':process_type,'max_character_limit_short':75,'max_character_limit_long':1000, 'temporal_resolution':res}
                __builtin__.run_details = run_details
                if res == 'H':
                    frequency = 'H'
                    desired_resolution_path = 'hourly'
                    descriptive_res = 'hours'
                elif res == 'D':
                    frequency = 'D'
                    desired_resolution_path = 'daily'
                    descriptive_res = 'days'
                elif res == 'M':
                    frequency = 'MS'
                    desired_resolution_path = 'monthly'
                    descriptive_res = 'months'
                __builtin__.desired_resolution_path = desired_resolution_path
                __builtin__.descriptive_res = descriptive_res

                #CREATE LIST OF ALL MINUTES IN TEMPORAL WINDOW (REPRESENTING START OF TEMPORAL WINDOW), WITH SPACING IN DEFINED TEMPORAL RESOLUTION BETWEEN START_DATE AND END_DATE
                start_dt = datetime.datetime(int(start_date[:4]),int(start_date[4:]),1,0,0)
                end_dt = datetime.datetime(int(end_date[:4]),int(end_date[4:]),1,0,0)
                all_dt = pd.date_range(start_dt, end_dt, freq=frequency).to_pydatetime()
                grid_start_minutes = [crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(dt) for dt in all_dt]
                __builtin__.grid_start_minutes = grid_start_minutes

                #TEST FOR PRESENCE OF TEMPORARY GRIDDED STATION DATA FILE DIRECTORY. IF DOES NOT EXIST CREATE IT
                temp_file_dir = '/esarchive/obs/uniyork/%s/%s/%s/temporary/'%(network,desired_resolution_path,parameter_details['bsc_parameter_name'])
                does_temporary_exist = os.path.isdir(temp_file_dir)
                if does_temporary_exist == False:
                    os.makedirs(temp_file_dir)
                
                #TEST FOR PRESENCE OF YEAR/MONTH TEMPORARY DIRECTORIES. IF DO NOT EXIST, CREATE THEM
                process_years = range(int(start_date[:4]),int(end_date[:4])+1)
                for year in process_years:
                
                    #is it first process year?
                    if year == process_years[0]:
                        months = np.arange(int(start_date[4:]), 13)
                    
                    #is it last process year?
                    elif year == process_years[-1]:
                        months = np.arange(1,int(end_date[4:]))
                    
                    #full year of months otherwise
                    else:
                        months = np.arange(1,13)
                    
                    for month in months:
                        if month < 10:
                            month_string = '0'+str(month)
                        else:
                            month_string = str(month)
                        temp_file_dir = '/esarchive/obs/uniyork/%s/%s/%s/temporary/%s/%s/'%(network,desired_resolution_path,parameter_details['bsc_parameter_name'],year,month_string)
                        does_temporary_exist = os.path.isdir(temp_file_dir)
                        if does_temporary_exist == False:
                            os.makedirs(temp_file_dir)
                        #DELETE ALL CURRENT TEMPORARY STATION DATA FILES FOR PARAMETER/NETWORK (IN CASE OF FAILED PREVIOUS RUN)
                        for zippath in glob.iglob(os.path.join(temp_file_dir, '*.npz')):
                            os.remove(zippath)

                #GET ALL PROCESSED NATIVE RESOLUTION STATION FILES BY NETWORK/PARAMETER
                station_files = sorted(glob.glob('/esarchive/obs/uniyork/%s/native/%s/*.npz'%(network,parameter_details['bsc_parameter_name'])))
                #station_files = [station_files[7]]
                #station_files = [station_files[16]]
                #HOW MANY STATIONS TO PROCESS?
                print 'NUMBER OF STATIONS: %s'%(len(station_files))
        
                ###------------------------------------------------------------------------------------###
                #TEMPORALLY GRID STATION SPECIFIC DATA AND OUTPUT TO TEMPORARY .npz FILES BY YEAR/MONTH
                ###------------------------------------------------------------------------------------###
                
                #SERIAL PROCESS
                if process_type == 'SERIAL':
                    for station_file in station_files:
                        crossnetwork_processing_modules.temporally_grid_data(station_file)   
        
                #PARALLEL PROCESS
                if 'PARALLEL' in run_details['process_type']:
                    crossnetwork_processing_modules.run_in_parallel(crossnetwork_processing_modules.temporally_grid_data, station_files)
                                
                ###------------------------------------------------------------------------------------###  
                #MERGE TEMPORARY STATION SPECIFIC .npz FILES TO 1 NETCDF BY YEAR/MONTH        
                ###------------------------------------------------------------------------------------###
            
                files_to_process = crossnetwork_processing_modules.get_numpy_files_per_month()
        
                #IS THERE DATA REMAINING TO PROCESS TO NETCDF?
                if len(files_to_process) == 0:
                    print 'NO VALID STATION DATA TO WRITE TO ANY MONTHLY NETCDF IN DESIRED TEMPORAL RANGE'
                else:
                    #SERIAL PROCESS
                    if process_type == 'SERIAL':
                        for ii in range(len(files_to_process)):
                            crossnetwork_processing_modules.write_final_merged_bsc_netcdf_by_month(files_to_process[ii])
                
                    #PARALLEL PROCESS
                    if 'PARALLEL' in run_details['process_type']:
                        crossnetwork_processing_modules.run_in_parallel(crossnetwork_processing_modules.write_final_merged_bsc_netcdf_by_month, files_to_process)
                
                #STOP TIMER
                t1 = time.time()
                total = t1-t0
                print '%s seconds'%(total)
                
                #DO GARBAGE COLLECTION
                gc.collect()
                