#!/usr/bin/env python
# -*- coding: utf-8 -*-

#WRITTEN BY DENE BOWDALO

###------------------------------------------------------------------------------------###
###IMPORT MODULES

###------------------------------------------------------------------------------------###

import __builtin__
from collections import Counter
import crossnetwork_processing_modules
import csv
import datetime
import gc
import glob
import itertools
from netCDF4 import Dataset, num2date, date2num
import math
import numpy as np
import os
import pandas as pd
import pytz
import time

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###

#MODULE FOR PROCESSING RAW DATA BY INTO A PROCESSED .npz FILE FOR EACH STATION IN NETWORK, FOR EACH SEPARATE PARAMETER 
#IN THIS PROCESS ALL DATA/METADATA IS STANDARDISED, AND DATA IS PASSED THROUGH MULTIPLE QUALITY CONTROL CHECKS
#TEMPORAL INFORMATION IS RETAINED AS NATIVELY REPORTED

#INPUT OPTIONS:

#NETWORK (list of network processed files that you want to convert) 
#AIRBASE, AQS, CAPMON, CASTNET, EANET, EBAS, EIONET, GAW, NAPS, SEARCH

#PARAMETERS (list of parameters that you wish to process)
#Gases:                     O3, NO, NO2, CO, SO2, NH3, HNO3, ISOPRENE
#Aerosol - bulk mass:       PM10, PM2.5, PM1 
#Aerosol - bulk speciated:  Al, As, BC, C, C_corrected, Ca++, Cd, Cl-, Co, Cr, Cu, EC, Fe, Hg, K+, Mg++, Mn, Na+, NH4+, Ni, NO3-, OC, OC_corrected, Pb, Se, SO4--, SO4--_NSS, V, Zn
#Aerosol - PM10 speciated:  PM10_Al, PM10_As, PM10_BC, PM10_C, PM10_C_corrected, PM10_Ca++, PM10_Cd, PM10_Cl-, PM10_Co, PM10_Cr, PM10_Cu, PM10_EC, PM10_Fe, PM10_Hg, PM10_K+, PM10_Mg++, PM10_Mn, PM10_Na+, PM10_NH4+, PM10_Ni, PM10_NO3-, PM10_OC, PM10_OC_corrected, PM10_Pb, PM10_Se, PM10_SO4--, PM10_SO4--_NSS, PM10_V, PM10_Zn 
#Aerosol - PM2.5 speciated: PM2.5_Al, PM2.5_As, PM2.5_BC, PM2.5_C, PM2.5_C_corrected, PM2.5_Ca++, PM2.5_Cd, PM2.5_Cl-, PM2.5_Co, PM2.5_Cr, PM2.5_Cu, PM2.5_EC, PM2.5_Fe, PM2.5_Hg, PM2.5_K+, PM2.5_Mg++, PM2.5_Mn, PM2.5_Na+, PM2.5_NH3, PM2.5_NH4+, PM2.5_Ni, PM2.5_NO3-, PM2.5_OC, PM2.5_OC_corrected, PM2.5_Pb, PM2.5_Se, PM2.5_SO4--, PM2.5_SO4--_NSS, PM2.5_V, PM2.5_Zn
#Aerosol - PM1 speciated:   PM1_Al, PM1_As, PM1_BC, PM1_C, PM1_C_corrected, PM1_Ca++, PM1_Cd, PM1_Cl-, PM1_Co, PM1_Cr, PM1_Cu, PM1_EC, PM1_Fe, PM1_Hg, PM1_K+, PM1_Mg++, PM1_Mn, PM1_Na+, PM1_NH4+, PM1_Ni, PM1_NO3-, PM1_OC, PM1_OC_corrected, PM1_Pb, PM1_Se, PM1_SO4--, PM1_SO4--_NSS, PM1_V, PM1_Zn 

#START_YEAR = YYYY (start year of processed data file - should be fixed as 1970) **DO NOT MODIFY
#END_YEAR = YYYY (end year of processed data file - should be year which contains last available data for network)

#PROCESS_TYPE = SERIAL, PARALLELN (where N = number of processes to start)

#*****All parameters will be processed first for one network

###------------------------------------------------------------------------------------###

networks = ['EBAS']
parameters = ['O3']
#parameters = ['O3','NO','NO2','CO','SO2','NH3','HNO3','ISOPRENE']
#parameters = ['PM10','PM2.5','PM1']
#parameters = ['C','Cl-','EC','Na+','NH4+','NO3-','OC','SO4--']
#parameters = ['PM10_C','PM10_Cl-','PM10_EC','PM10_Na+','PM10_NH4+','PM10_NO3-','PM10_OC','PM10_SO4--']
#parameters = ['PM2.5_C','PM2.5_Cl-','PM2.5_EC','PM2.5_Na+','PM2.5_NH4+','PM2.5_NO3-','PM2.5_OC','PM2.5_SO4--']
#parameters = ['PM1_C','PM1_Cl-','PM1_EC','PM1_Na+','PM1_NH4+','PM1_NO3-','PM1_OC','PM1_SO4--']
start_year = '1970'
end_year = '2018'
process_type = 'PARALLEL16'

###------------------------------------------------------------------------------------###
if __name__ == '__main__':
    for network in networks:
        for parameter in parameters:
            #START TIMER
            t0 = time.time()
            
            #WHICH NETWORK, WHICH PARAMETER?
            print network, parameter
            
            ###----------------------------------------------------------------------------###
            #SAVE CORE RUN DETAILS TO GLOBAL DICTIONARY
            ###----------------------------------------------------------------------------###
            
            run_details = {'network':network,'parameter':parameter,'start_year':start_year,'end_year':end_year,'process_type':process_type, 'process_mode':process_mode}
            __builtin__.run_details = run_details
            
            ###----------------------------------------------------------------------------###
            #IMPORT SPECIFIC NETWORK PROCESSING MODULE FOR NETWORK BEING PROCESSED
            ###----------------------------------------------------------------------------###
    
            network_processor = __import__(run_details['network']+'_processor')
    
            ###----------------------------------------------------------------------------###
            #GET SPECIFIC PARAMETER AND NETWORK ASSOCIATED VARIABLES 
            ###----------------------------------------------------------------------------###
    
            parameter_details = crossnetwork_processing_modules.get_parameter_details()[run_details['parameter']]
            network_details = crossnetwork_processing_modules.get_network_details()
            #MAKE CORE PARAMETER DATA GLOBAL ACROSS ALL SUBMODULES
            __builtin__.parameter_details = parameter_details
            __builtin__.network_details = network_details
        
            ###----------------------------------------------------------------------------###
            #CREATE/CLEAN TEMPORARY DIRECTORIES AND FILES USED IN PROCESSING
             ###----------------------------------------------------------------------------###
            
            #TEST FOR PRESENCE OF TEMPORARY STATION DATA FILE DIRECTORY. USED TO STORE INITIAL NUMPY .npz FILES OF RAW DATA BEFORE QA CHECKS.
            #IF DOES NOT EXIST CREATE IT
            temp_file_dir = '/esarchive/obs/uniyork/%s/native/%s/temporary/'%(run_details['network'],parameter_details['bsc_parameter_name'])
            does_temporary_exist = os.path.isdir(temp_file_dir)
            if does_temporary_exist == False:
                os.makedirs(temp_file_dir)
            #DELETE ALL CURRENT TEMPORARY STATION DATA FILES FOR PARAMETER/NETWORK (IN CASE OF FAILED PREVIOUS RUN AND HAVE NOT BEEN CLEANED)
            for zippath in glob.iglob(os.path.join(temp_file_dir, '*.npz')):
                os.remove(zippath)
                
            #TEST FOR PRESENCE OF PARALLEL SAFE DIRECTORY. USED FOR KEEPING TRACK OF WHICH STATIONS HAVE ALREADY BEEN PROCESSED, IN CASE OF SPORADIC FAILING OF MULTIPROCESSING MODULE - TO PREVENT REPEATING WORK 
            #IF DOES NOT EXIST CREATE IT
            parallel_safe_dir = '/esarchive/obs/uniyork/%s/native/%s/parallel_safe/'%(run_details['network'],parameter_details['bsc_parameter_name'])
            does_temporary_exist = os.path.isdir(parallel_safe_dir)
            if does_temporary_exist == False:
                os.makedirs(parallel_safe_dir)
            #DELETE ALL CURRENT PARALLEL SAFE STATION FILES PARAMETER/NETWORK (IN CASE OF FAILED PREVIOUS RUN AND HAVE NOT BEEN CLEANED)
            for current_file in glob.glob('%s*'%(parallel_safe_dir)):
                os.remove(current_file)
                
            #TEST FOR PRESENCE OF FINAL STATION PROCESS INFORMATION DIRECTORY. USED FOR GETTING ALL VALID FULLY PROCESSED STATION REFERENCES 
            #IF DOES NOT EXIST CREATE IT
            final_file_process_dir = '/esarchive/obs/uniyork/%s/native/%s/final_process_valid/'%(run_details['network'],parameter_details['bsc_parameter_name'])
            does_temporary_exist = os.path.isdir(final_file_process_dir)
            if does_temporary_exist == False:
                os.makedirs(final_file_process_dir)
            #DELETE ALL CURRENT FINAL STATION FILES FOR PARAMETER/NETWORK (IN CASE OF FAILED PREVIOUS RUN AND HAVE NOT BEEN CLEANED)
            for current_file in glob.glob('%s*'%(final_file_process_dir)):
                os.remove(current_file)
        
            ###----------------------------------------------------------------------------###
            #PROCESSING OF OBSERVATIONS FROM RAW FILES BEGINS HERE
            
            #STAGE 1. PARSE ALL METADATA/DATA FROM DATA FILES FOR NETWORK/PARAMETER INTO NUMPY .npz FILES PER STATION 
            
            #THE WAY OBSERVATIONS ARE READ IS HANDLED DIFFERENTLY DEPENDENT ON HOW DATA IS REPORTED BY THE NETWORK.  
            #DATA IS REPORTED IN 2 WAYS: 
            #1. NETWORKS WHICH GIVE DATA PER SITE PER YEAR
            #2. NETWORKS WHICH GIVE DATA FOR ALL SITES PER YEAR     
            ###----------------------------------------------------------------------------###

            #GET ALL DATA FILES FOR SPECIFIC PARAMETER, FOR NETWORK 
            data_files = crossnetwork_processing_modules.get_data_files()
        
            #BRANCH HERE, DEPENDENT ON HOW OBSERVATIONS ARE REPORTED
            #1. IF FILE STORAGE TYPE IS ALL SITES PER YEAR, PROCESS ALL STATIONS YEAR BY YEAR
            #2. IF FILE STORAGE TYPE IS SITE PER YEAR, PROCESS EACH ALL YEARS STATION BY STATION
        
            #GET ALL DIFFERENT YEARS OF FILENAMES DATA IS SEPARATED BETWEEN
            if network_details['file_storage_type'] == 'all_per_year':
                data_years = range(int(run_details['start_year']),int(run_details['end_year']))
            elif network_details['file_storage_type'] == 'site_per_year':
                data_years = range(1)
        
            #ITERATE THROUGH ALL YEARS OF DATA FILES (JUST 1 YEAR IF HAVE SITES REPORTED PER YEAR)
            for year in data_years:
            
                #IF FILE STORAGE TYPE IS ALL SITES PER YEAR, LIMIT DATA FILES TO RELEVANT YEARLY FILE
                if network_details['file_storage_type'] == 'all_per_year':
                    data_files_to_process = [s for s in data_files if str(year) in s]
                    print 'PROCESSING YEAR: %s'%(year)
                #OTHERWISE DO NOT FILTER FILES BY YEAR
                elif network_details['file_storage_type'] == 'site_per_year':
                    data_files_to_process = data_files
                
                #GET ALL STATION REFERENCES & REFERENCES TO CONNECT METADATA/DATA FROM DATA FILE/S
                station_references = crossnetwork_processing_modules.get_station_references(data_files_to_process) 
                #station_references = station_references[51:]
                #station_references = station_references[:3]
                
                #HOW MANY STATIONS TO PROCESS?
                print 'NUMBER OF STATIONS: %s'%(len(station_references))
            
                #LOAD EXTERNAL METADATA INTO MEMORY (*IF EXISTS FOR NETWORK)        
                if network_details['external_metadata_file'] != '':
                    external_metadata = crossnetwork_processing_modules.read_external_metadata(station_references)
                else:
                    external_metadata = [[] for _ in xrange(len(station_references))]
            
                #IF FILE STORAGE TYPE IS SITE PER YEAR, SPLIT DATA FILES INTO LIST OF LISTS CONTAINING STATION SPECIFIC DATA FILES
                if network_details['file_storage_type'] == 'site_per_year':
                    data_files_to_process = crossnetwork_processing_modules.generate_data_files_LoL(station_references,data_files_to_process)
        
                #ZIP VARIABLES THAT WILL BE PROCESSED
                zipped_list = zip(range(len(station_references)),station_references,data_files_to_process,external_metadata)
            
                #UPDATE GLOBAL PROCESS STAGE
                process_stage = 'first_write'
                __builtin__.process_stage = process_stage
              
                #SERIAL PROCESS
                if run_details['process_type'] == 'SERIAL':
                    for ref_i in range(len(station_references)):
                        network_processor.read_data(zipped_list[ref_i])
                    
                #PARALLEL PROCESS (RUN USING PARALLEL SAFE MODULE - AS SPORADICALLY WILL BREAK DUE TO STRUCTERROR ERROR - UNKNOWN WHY THIS HAPPENS) 
                if 'PARALLEL' in run_details['process_type']:
                    crossnetwork_processing_modules.run_parallel_safe(network_processor.read_data, zipped_list, parallel_safe_dir)
                    
                    #CLEAN PARALLEL SAFE DIRECTORY
                    for current_file in glob.glob('%s*'%(parallel_safe_dir)):
                        os.remove(current_file)
                        
            #CHECK IF HAVE VALID STATIONS?
            if len(station_references) == 0:
                break 
                    
            ###----------------------------------------------------------------------------###
            #STAGE 2. GET PRIMARY PROCESSED REFERENCES. CHECK EACH STATION HAS 1 SPECIFIC FILE.
            #         IF NOT, DEAL WITH THIS
            ###----------------------------------------------------------------------------###
        
            #GET PROCESSED REFERENCES - GET REFERENCES FROM OUTPUT NUMPY FILES
            #THESE DIFFER FROM INITIAL STATION REFERENCES USED BEFORE, AS SOME OF THE STATION REFERENCES MAY NO LONGER BE VALID - AS COULD HAVE HAD CORRUPT/BADLY FORMATTED/OR NO DATA  
            processed_references = crossnetwork_processing_modules.get_processed_references()
            
            #DO CHECK TO DEAL WITH NUMPY STATION SPECIFIC FILES THAT HAVE DUPLICATES
            #(THIS HAPPENS IN INSTANCES WHERE STATION CODES ARE UNKNOWN UNTIL READING FILES)
            base_references = []
            #find all instances of duplicated temporary files
            for reference in processed_references:
                base_references.append(reference.split('_dup')[0])
            duplicated_references = sorted([item for item, count in Counter(base_references).items() if count > 1])
            
            if len(duplicated_references) > 0:
                #SERIAL PROCESS
                if run_details['process_type'] == 'SERIAL':
                    for ref in duplicated_references:
                        crossnetwork_processing_modules.resolve_duplicate_reference_files(ref)
                #PARALLEL PROCESS
                if 'PARALLEL' in run_details['process_type']:
                    crossnetwork_processing_modules.run_in_parallel(crossnetwork_processing_modules.resolve_duplicate_reference_files, duplicated_references)
            
            validated_processed_references = np.unique(base_references)
            
            ###----------------------------------------------------------------------------###
            #STAGE 3. PASS ALL PROCESSED NUMPY FILES PER STATION THROUGH QA
            ###----------------------------------------------------------------------------###
            
            #UPDATE GLOBAL PROCESS STAGE
            __builtin__.process_stage = 'final_write'
        
            #SERIAL PROCESS
            if run_details['process_type'] == 'SERIAL':
                all_modal_metadata = []
                for ref in validated_processed_references:
                    modal_metadata = crossnetwork_processing_modules.final_processing(ref)   
                    all_modal_metadata.append(modal_metadata)
            
            #PARALLEL PROCESS
            if 'PARALLEL' in run_details['process_type']:
                all_modal_metadata = crossnetwork_processing_modules.run_in_parallel(crossnetwork_processing_modules.final_processing, validated_processed_references)
                
            #GET LIST OF FINAL PROCESSED REFERENCES FROM WRITTEN FINAL STATION FILES DIRECTORY 
            #THESE AGAIN DIFFER FROM REFERENCES USED BEFORE, AS SOME OF THE STATION REFERENCES MAY NOW BE SPLIT OUT (DUE TO CHANGING KEY METADATA), OR SOME MAY NOT EXIST ANY MORE (DUE TO MISSING KEY METADATA)
            final_processed_references = []
            for file in glob.glob('%s*'%(final_file_process_dir)):
                final_processed_references.append(file.split('/')[-1])
                
            #CLEAN ALL FINAL PROCESSED REFERENCE DATA FILES.
            for current_file in glob.glob('%s*'%(final_file_process_dir)):
                os.remove(current_file)
            
            ###----------------------------------------------------------------------------###
            #STAGE 4. WRITE RETURNED MODAL METADATA FOR ALL STATIONS TO STATIC CSV FILE FOR NETWORK
            ###----------------------------------------------------------------------------###
            
            crossnetwork_processing_modules.write_static_modal_metadata(all_modal_metadata)
                
            #STOP TIMER
            t1 = time.time()
            total = t1-t0
            print '%s seconds'%(total)
            
            #DO GARBAGE COLLECTION
            gc.collect()
