From e80c19b9c936b5e55b23b805963683016de4c780 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 18 Jul 2019 16:16:06 +0200 Subject: [PATCH 1/2] - Shipping port: Parallelized --- conf/hermes.conf | 9 +++--- hermesv3_bu/sectors/shipping_port_sector.py | 35 +++++++++++++++------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 7d7120e..c84a48f 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMES__traff_area.nc +output_name = HERMES__ships_new_2.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -114,12 +114,13 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver #################################################################### # SECTORS #################################################################### +[SECTOR MANAGEMENT] writing_processors = 4 do_aviation = 0 aviation_processors = 4 -do_shipping_port = 0 -shipping_port_processors = 1 +do_shipping_port = 1 +shipping_port_processors = 2 do_livestock = 0 livestock_processors = 1 do_crop_operations = 0 @@ -132,7 +133,7 @@ do_residential = 0 residential_processors = 4 do_recreational_boats = 0 recreational_boats_processors = 1 -do_point_sources = 1 +do_point_sources = 0 point_sources_processors = 1 do_traffic = 0 traffic_processors = 4 diff --git a/hermesv3_bu/sectors/shipping_port_sector.py b/hermesv3_bu/sectors/shipping_port_sector.py index da880b6..bb6823b 100755 --- a/hermesv3_bu/sectors/shipping_port_sector.py +++ b/hermesv3_bu/sectors/shipping_port_sector.py @@ -6,6 +6,7 @@ import geopandas as gpd import numpy as np import timeit from hermesv3_bu.logger.log import Log +from hermesv3_bu.io_server.io_shapefile import IoShapefile class ShippingPortSector(Sector): @@ -81,9 +82,6 @@ class ShippingPortSector(Sector): spent_time = timeit.default_timer() logger.write_log('===== SHIPPING PORT SECTOR =====') - if comm.Get_size() > 1: - raise ValueError('Shipping port sector is not parallelised set shipping_port_processors to 1') - super(ShippingPortSector, self).__init__( comm, logger, auxiliary_dir, grid_shp, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -92,17 +90,12 @@ class ShippingPortSector(Sector): self.ef_engine = self.read_profiles(ef_dir) self.vessel_list = vessel_list - if port_list is not None: - unknown_ports = [x for x in port_list if x not in list(pd.read_csv(tonnage_path).code.values)] - if len(unknown_ports) > 0: - raise ValueError("The port(s) {0} are not listed on the {1} file.".format(unknown_ports, tonnage_path)) - self.port_list = port_list - else: - self.port_list = list(self.read_profiles(tonnage_path).code.values) self.hoteling_shapefile_path = hoteling_shapefile_path self.maneuvering_shapefile_path = maneuvering_shapefile_path + self.port_list = self.get_port_list() + self.engine_percent = self.read_profiles(engine_percent_path) self.tonnage = self.read_profiles(tonnage_path) self.tonnage.set_index('code', inplace=True) @@ -110,6 +103,28 @@ class ShippingPortSector(Sector): self.power_values = self.read_profiles(power_path) self.logger.write_time_log('ShippingPortSector', '__init__', timeit.default_timer() - spent_time) + def get_port_list(self): + if self.comm.Get_rank() == 0: + port_shp = IoShapefile(self.comm).read_shapefile_serial(self.maneuvering_shapefile_path) + port_shp.drop(columns=['Name', 'Weight'], inplace=True) + + port_shp = gpd.sjoin(port_shp, self.clip.shapefile.to_crs(port_shp.crs), how='inner', op='intersects') + port_list = np.unique(port_shp['code'].values) + print port_list + if len(port_list) < self.comm.Get_size(): + raise ValueError("The chosen number of processors {0} exceeds the number of involved ports {1}.".format( + self.comm.Get_size(), len(port_list)) + " Set {0} at shipping_port_processors value.".format( + len(port_list))) + port_list = np.array_split(port_list, self.comm.Get_size()) + else: + port_list = None + + port_list = self.comm.scatter(port_list, root=0) + + if len(port_list) == 0: + raise ValueError("The number ") + return list(port_list) + def read_monthly_profiles(self, path): """ Read the DataFrame of the monthly profiles with the month number as columns. -- GitLab From 9f0ce508eaa8077a500095057763041e08445d4a Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 18 Jul 2019 16:38:50 +0200 Subject: [PATCH 2/2] - Configuration file: Drop do_. substitute by the number of processors --- conf/hermes.conf | 31 +++++++------------- hermesv3_bu/config/config.py | 56 ++++++++++++++---------------------- 2 files changed, 32 insertions(+), 55 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index c84a48f..05ee139 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -117,28 +117,17 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver [SECTOR MANAGEMENT] writing_processors = 4 -do_aviation = 0 -aviation_processors = 4 -do_shipping_port = 1 +aviation_processors = 0 shipping_port_processors = 2 -do_livestock = 0 -livestock_processors = 1 -do_crop_operations = 0 -crop_operations_processors = 1 -do_crop_fertilizers = 0 -crop_fertilizers_processors = 4 -do_agricultural_machinery = 0 -agricultural_machinery_processors = 1 -do_residential = 0 -residential_processors = 4 -do_recreational_boats = 0 -recreational_boats_processors = 1 -do_point_sources = 0 -point_sources_processors = 1 -do_traffic = 0 -traffic_processors = 4 -do_traffic_area = 0 -traffic_area_processors = 1 +livestock_processors = 0 +crop_operations_processors = 0 +crop_fertilizers_processors = 0 +agricultural_machinery_processors = 0 +residential_processors = 0 +recreational_boats_processors = 0 +point_sources_processors = 0 +traffic_processors = 0 +traffic_area_processors = 0 [SHAPEFILES] diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 84e7bd1..043ee0e 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -152,29 +152,17 @@ class Config(ArgParser): p.add_argument('--inc_lon', required=False, type=float, help='Longitude grid resolution.') # ===== SECTOR SELECTION ===== - p.add_argument('--do_traffic', required=False, type=str, default='True') - p.add_argument('--do_traffic_area', required=False, type=str, default='True') - p.add_argument('--do_aviation', required=False, type=str, default='True') - p.add_argument('--do_point_sources', required=False, type=str, default='True') - p.add_argument('--do_recreational_boats', required=False, type=str, default='True') - p.add_argument('--do_shipping_port', required=False, type=str, default='True') - p.add_argument('--do_residential', required=False, type=str, default='True') - p.add_argument('--do_livestock', required=False, type=str, default='True') - p.add_argument('--do_crop_operations', required=False, type=str, default='True') - p.add_argument('--do_crop_fertilizers', required=False, type=str, default='True') - p.add_argument('--do_agricultural_machinery', required=False, type=str, default='True') - - p.add_argument('--traffic_processors', required=False, type=int, default='True') - p.add_argument('--traffic_area_processors', required=False, type=int, default='True') - p.add_argument('--aviation_processors', required=False, type=int, default='True') - p.add_argument('--point_sources_processors', required=False, type=int, default='True') - p.add_argument('--recreational_boats_processors', required=False, type=int, default='True') - p.add_argument('--shipping_port_processors', required=False, type=int, default='True') - p.add_argument('--residential_processors', required=False, type=int, default='True') - p.add_argument('--livestock_processors', required=False, type=int, default='True') - p.add_argument('--crop_operations_processors', required=False, type=int, default='True') - p.add_argument('--crop_fertilizers_processors', required=False, type=int, default='True') - p.add_argument('--agricultural_machinery_processors', required=False, type=int, default='True') + p.add_argument('--traffic_processors', required=True, type=int) + p.add_argument('--traffic_area_processors', required=True, type=int) + p.add_argument('--aviation_processors', required=True, type=int) + p.add_argument('--point_sources_processors', required=True, type=int) + p.add_argument('--recreational_boats_processors', required=True, type=int) + p.add_argument('--shipping_port_processors', required=True, type=int) + p.add_argument('--residential_processors', required=True, type=int) + p.add_argument('--livestock_processors', required=True, type=int) + p.add_argument('--crop_operations_processors', required=True, type=int) + p.add_argument('--crop_fertilizers_processors', required=True, type=int) + p.add_argument('--agricultural_machinery_processors', required=True, type=int) p.add_argument('--speciation_map', required=False, help='...') @@ -411,17 +399,17 @@ class Config(ArgParser): comm.Barrier() self.create_dir(arguments.auxiliary_files_path) - arguments.do_traffic = self._parse_bool(arguments.do_traffic) - arguments.do_traffic_area = self._parse_bool(arguments.do_traffic_area) - arguments.do_aviation = self._parse_bool(arguments.do_aviation) - arguments.do_point_sources = self._parse_bool(arguments.do_point_sources) - arguments.do_recreational_boats = self._parse_bool(arguments.do_recreational_boats) - arguments.do_shipping_port = self._parse_bool(arguments.do_shipping_port) - arguments.do_residential = self._parse_bool(arguments.do_residential) - arguments.do_livestock = self._parse_bool(arguments.do_livestock) - arguments.do_crop_operations = self._parse_bool(arguments.do_crop_operations) - arguments.do_crop_fertilizers = self._parse_bool(arguments.do_crop_fertilizers) - arguments.do_agricultural_machinery = self._parse_bool(arguments.do_agricultural_machinery) + arguments.do_traffic = arguments.traffic_processors > 0 + arguments.do_traffic_area = arguments.traffic_area_processors > 0 + arguments.do_aviation = arguments.aviation_processors > 0 + arguments.do_point_sources = arguments.point_sources_processors > 0 + arguments.do_recreational_boats = arguments.recreational_boats_processors > 0 + arguments.do_shipping_port = arguments.shipping_port_processors > 0 + arguments.do_residential = arguments.residential_processors > 0 + arguments.do_livestock = arguments.livestock_processors > 0 + arguments.do_crop_operations = arguments.crop_operations_processors > 0 + arguments.do_crop_fertilizers = arguments.crop_fertilizers_processors > 0 + arguments.do_agricultural_machinery = arguments.agricultural_machinery_processors > 0 # Aviation lists arguments.airport_list = self._parse_list(arguments.airport_list) -- GitLab