diff --git a/conf/hermes.conf b/conf/hermes.conf index 059faa88d3bcf5b495dbf2b89bd77e4dd6b907c9..ad9ea14736848678cbfc107aa8a6a3e677db1d05 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -9,7 +9,8 @@ start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 output_timestep_num = 24 -auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ +auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/__test +first_time = 0 erase_auxiliary_files = 0 @@ -55,10 +56,10 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver #y_0 = 43862.90625 # CATALUNYA test - #nx = 28 - #ny = 30 - nx = 4 - ny = 4 + nx = 28 + ny = 30 + #nx = 4 + #ny = 4 inc_x = 10000 inc_y = 10000 x_0 = 253151.59375 @@ -118,17 +119,17 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver #################################################################### [SECTOR MANAGEMENT] writing_processors = 1 - -aviation_processors = 1 -shipping_port_processors = 1 -livestock_processors = 12 -crop_operations_processors = 1 -crop_fertilizers_processors = 4 -agricultural_machinery_processors = 1 -residential_processors = 4 -recreational_boats_processors = 4 -point_sources_processors = 16 -traffic_processors = 256 +# +# aviation_processors = 1 +# shipping_port_processors = 1 +# livestock_processors = 12 +# crop_operations_processors = 1 +# crop_fertilizers_processors = 4 +# agricultural_machinery_processors = 1 +# residential_processors = 4 +# recreational_boats_processors = 4 +# point_sources_processors = 16 +# traffic_processors = 256 traffic_area_processors = 1 [SHAPEFILES] diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 3091a86bdc9e7d94c0f1c5b54e30d9d767c6cb33..c3fccf8492de2b97a209465326b9aa747eec6cc6 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -10,13 +10,17 @@ class Config(ArgParser): """ Configuration arguments class. """ - def __init__(self, new_date=None): + def __init__(self, new_date=None, comm=None): """ Read and parse all the arguments. :param new_date: Starting date for simulation loop day. :type new_date: datetime.datetime """ + if comm is None: + comm = MPI.COMM_WORLD + self.comm = comm + self.new_date = new_date super(Config, self).__init__() @@ -55,6 +59,8 @@ class Config(ArgParser): p.add_argument('--auxiliary_files_path', required=True, help='Path to the directory where the necessary auxiliary files will be created if them are ' + 'not created yet.') + p.add_argument('--first_time', required=False, default='False', type=str, + help='Indicates if you want to run it for first time (only create auxiliary files).') p.add_argument('--erase_auxiliary_files', required=False, default='False', type=str, help='Indicates if you want to start from scratch removing the auxiliary files already created.') @@ -686,15 +692,15 @@ class Config(ArgParser): arguments.end_date = self._parse_end_date(arguments.end_date, arguments.start_date) arguments.output_name = self.get_output_name(arguments) + arguments.first_time = self._parse_bool(arguments.first_time) arguments.erase_auxiliary_files = self._parse_bool(arguments.erase_auxiliary_files) self.create_dir(arguments.output_dir) if arguments.erase_auxiliary_files: if os.path.exists(arguments.auxiliary_files_path): - comm = MPI.COMM_WORLD - if comm.Get_rank() == 0: + if self.comm.Get_rank() == 0: rmtree(arguments.auxiliary_files_path) - comm.Barrier() + self.comm.Barrier() self.create_dir(arguments.auxiliary_files_path) # Booleans @@ -797,8 +803,7 @@ class Config(ArgParser): full_path = os.path.join(arguments.output_dir, file_name) return full_path - @staticmethod - def create_dir(path): + def create_dir(self, path): """ Create the given folder if it is not created yet. @@ -807,8 +812,7 @@ class Config(ArgParser): """ import os from mpi4py import MPI - icomm = MPI.COMM_WORLD - comm = icomm.Split(color=0, key=0) + comm = self.comm.Split(color=0, key=0) rank = comm.Get_rank() if rank == 0: diff --git a/hermesv3_bu/hermes.py b/hermesv3_bu/hermes.py index 9101d5f201983e882af2438a39a395e78b35cd13..e35595982c0d0261d5c1323ef6741afab6ee30f5 100755 --- a/hermesv3_bu/hermes.py +++ b/hermesv3_bu/hermes.py @@ -7,7 +7,7 @@ from mpi4py import MPI from datetime import timedelta from hermesv3_bu.config.config import Config -from hermesv3_bu.grids.grid import select_grid +from hermesv3_bu.grids.grid import select_grid, Grid from hermesv3_bu.clipping.clip import select_clip from hermesv3_bu.writer.writer import select_writer from hermesv3_bu.sectors.sector_manager import SectorManager @@ -18,9 +18,19 @@ class Hermes(object): """ Interface class for HERMESv3. """ - def __init__(self, config): - self.initial_time = timeit.default_timer() - self.comm = MPI.COMM_WORLD + def __init__(self, config, comm=None): + """ + + :param config: Configuration file object + :type config: Config + + :param comm: Communicator + :type comm: MPI.Comm + """ + self.__initial_time = timeit.default_timer() + if comm is None: + comm = MPI.COMM_WORLD + self.comm = comm self.arguments = config.arguments self.logger = Log(self.arguments) @@ -40,7 +50,7 @@ class Hermes(object): self.writer = select_writer(self.logger, self.arguments, self.grid, self.date_array) - self.logger.write_time_log('Hermes', '__init__', timeit.default_timer() - self.initial_time) + self.logger.write_time_log('Hermes', '__init__', timeit.default_timer() - self.__initial_time) def main(self): """ @@ -48,17 +58,20 @@ class Hermes(object): """ from datetime import timedelta - emis = self.sector_manager.run() - waiting_time = timeit.default_timer() - self.comm.Barrier() - self.logger.write_log('All emissions calculated!') - self.logger.write_time_log('Hermes', 'Waiting_to_write', timeit.default_timer() - waiting_time) + if self.arguments.first_time: + self.logger.write_log('***** HERMESv3_BU First Time finished successfully *****') + else: + emis = self.sector_manager.run() + waiting_time = timeit.default_timer() + self.comm.Barrier() + self.logger.write_log('All emissions calculated!') + self.logger.write_time_log('Hermes', 'Waiting_to_write', timeit.default_timer() - waiting_time) - self.writer.write(emis) - self.comm.Barrier() + self.writer.write(emis) + self.comm.Barrier() - self.logger.write_log('***** HERMESv3_BU simulation finished successfully *****') - self.logger.write_time_log('Hermes', 'TOTAL', timeit.default_timer() - self.initial_time) + self.logger.write_log('***** HERMESv3_BU simulation finished successfully *****') + self.logger.write_time_log('Hermes', 'TOTAL', timeit.default_timer() - self.__initial_time) self.logger.finish_logs() if self.arguments.start_date < self.arguments.end_date: @@ -67,8 +80,8 @@ class Hermes(object): return None -def run(): - date = Hermes(Config()).main() +def run(comm=None): + date = Hermes(Config(comm), comm).main() while date is not None: date = Hermes(Config(new_date=date)).main() sys.exit(0) diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index a2a79f843dc6d5200dc533298c238fd4758ea0ba..7bb7d0ec036239170532c18719bccc8931915dab 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -26,11 +26,11 @@ class SectorManager(object): :type arguments: NameSpace """ spent_time = timeit.default_timer() - self.logger = logger + self.__logger = logger self.sector_list = self.make_sector_list(arguments, comm_world.Get_size()) - self.logger.write_log('Sector process distribution:') + self.__logger.write_log('Sector process distribution:') for sect, procs in self.sector_list.items(): - self.logger.write_log('\t{0}: {1}'.format(sect, procs)) + self.__logger.write_log('\t{0}: {1}'.format(sect, procs)) color = 10 agg_color = 99 @@ -38,7 +38,7 @@ class SectorManager(object): if sector == 'aviation' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.aviation_sector import AviationSector self.sector = AviationSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.aviation_source_pollutants, grid.vertical_desctiption, arguments.airport_list, arguments.plane_list, arguments.airport_shapefile_path, arguments.airport_runways_shapefile_path, @@ -50,7 +50,7 @@ class SectorManager(object): elif sector == 'shipping_port' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.shipping_port_sector import ShippingPortSector self.sector = ShippingPortSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.shipping_port_source_pollutants, grid.vertical_desctiption, arguments.vessel_list, arguments.port_list, arguments.hoteling_shapefile_path, arguments.maneuvering_shapefile_path, @@ -63,7 +63,7 @@ class SectorManager(object): elif sector == 'livestock' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.livestock_sector import LivestockSector self.sector = LivestockSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.livestock_source_pollutants, grid.vertical_desctiption, arguments.animal_list, arguments.gridded_livestock, arguments.correction_split_factors, @@ -128,7 +128,7 @@ class SectorManager(object): elif sector == 'residential' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.residential_sector import ResidentialSector self.sector = ResidentialSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.residential_source_pollutants, grid.vertical_desctiption, arguments.fuel_list, arguments.nuts3_shapefile, arguments.nuts2_shapefile, arguments.population_density_map, @@ -142,7 +142,7 @@ class SectorManager(object): elif sector == 'recreational_boats' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.recreational_boats_sector import RecreationalBoatsSector self.sector = RecreationalBoatsSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.recreational_boats_source_pollutants, grid.vertical_desctiption, arguments.recreational_boats_list, arguments.recreational_boats_density_map, @@ -154,7 +154,7 @@ class SectorManager(object): elif sector == 'point_sources' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.point_source_sector import PointSourceSector self.sector = PointSourceSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.point_source_pollutants, grid.vertical_desctiption, arguments.point_source_catalog, arguments.point_source_monthly_profiles, @@ -175,7 +175,7 @@ class SectorManager(object): elif sector == 'traffic' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.traffic_sector import TrafficSector self.sector = TrafficSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.traffic_pollutants, grid.vertical_desctiption, arguments.road_link_path, arguments.fleet_compo_path, arguments.traffic_speed_hourly_path, arguments.traffic_monthly_profiles, @@ -193,7 +193,7 @@ class SectorManager(object): elif sector == 'traffic_area' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.traffic_area_sector import TrafficAreaSector self.sector = TrafficAreaSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.traffic_area_pollutants, grid.vertical_desctiption, arguments.population_density_map, arguments.speciation_map, arguments.molecular_weights, arguments.do_evaporative, arguments.traffic_area_gas_path, @@ -207,7 +207,7 @@ class SectorManager(object): elif sector == 'solvents' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.solvents_sector import SolventsSector self.sector = SolventsSector( - comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, + comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.__logger, arguments.auxiliary_files_path, grid, clip, date_array, arguments.solvents_pollutants, grid.vertical_desctiption, arguments.speciation_map, arguments.molecular_weights, arguments.solvents_speciation_profiles, arguments.solvents_monthly_profile, @@ -219,12 +219,12 @@ class SectorManager(object): color += 1 - self.logger.write_time_log('SectorManager', '__init__', timeit.default_timer() - spent_time) + self.__logger.write_time_log('SectorManager', '__init__', timeit.default_timer() - spent_time) def run(self): spent_time = timeit.default_timer() emis = self.sector.calculate_emissions() - self.logger.write_time_log('SectorManager', 'run', timeit.default_timer() - spent_time) + self.__logger.write_time_log('SectorManager', 'run', timeit.default_timer() - spent_time) return emis def make_sector_list(self, arguments, max_procs): @@ -242,5 +242,5 @@ class SectorManager(object): "'{0}': {1}".format( accum, {sector: len(sector_procs) for sector, sector_procs in sector_dict.items()})) - self.logger.write_time_log('SectorManager', 'make_sector_list', timeit.default_timer() - spent_time) + self.__logger.write_time_log('SectorManager', 'make_sector_list', timeit.default_timer() - spent_time) return sector_dict