diff --git a/conf/hermes.conf b/conf/hermes.conf index 7d7120ea99210ad9f1c45286bacff4062f80bce0..05ee139264a51505d7a1a690a4436d2a0012a4b7 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMES__traff_area.nc +output_name = HERMES__ships_new_2.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -114,30 +114,20 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver #################################################################### # SECTORS #################################################################### +[SECTOR MANAGEMENT] writing_processors = 4 -do_aviation = 0 -aviation_processors = 4 -do_shipping_port = 0 -shipping_port_processors = 1 -do_livestock = 0 -livestock_processors = 1 -do_crop_operations = 0 -crop_operations_processors = 1 -do_crop_fertilizers = 0 -crop_fertilizers_processors = 4 -do_agricultural_machinery = 0 -agricultural_machinery_processors = 1 -do_residential = 0 -residential_processors = 4 -do_recreational_boats = 0 -recreational_boats_processors = 1 -do_point_sources = 1 -point_sources_processors = 1 -do_traffic = 0 -traffic_processors = 4 -do_traffic_area = 0 -traffic_area_processors = 1 +aviation_processors = 0 +shipping_port_processors = 2 +livestock_processors = 0 +crop_operations_processors = 0 +crop_fertilizers_processors = 0 +agricultural_machinery_processors = 0 +residential_processors = 0 +recreational_boats_processors = 0 +point_sources_processors = 0 +traffic_processors = 0 +traffic_area_processors = 0 [SHAPEFILES] diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 84e7bd1faa91dd891346db32811ceeb072f13f2e..043ee0e4acc824eb254d1593529bcbe6a1e5eb85 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -152,29 +152,17 @@ class Config(ArgParser): p.add_argument('--inc_lon', required=False, type=float, help='Longitude grid resolution.') # ===== SECTOR SELECTION ===== - p.add_argument('--do_traffic', required=False, type=str, default='True') - p.add_argument('--do_traffic_area', required=False, type=str, default='True') - p.add_argument('--do_aviation', required=False, type=str, default='True') - p.add_argument('--do_point_sources', required=False, type=str, default='True') - p.add_argument('--do_recreational_boats', required=False, type=str, default='True') - p.add_argument('--do_shipping_port', required=False, type=str, default='True') - p.add_argument('--do_residential', required=False, type=str, default='True') - p.add_argument('--do_livestock', required=False, type=str, default='True') - p.add_argument('--do_crop_operations', required=False, type=str, default='True') - p.add_argument('--do_crop_fertilizers', required=False, type=str, default='True') - p.add_argument('--do_agricultural_machinery', required=False, type=str, default='True') - - p.add_argument('--traffic_processors', required=False, type=int, default='True') - p.add_argument('--traffic_area_processors', required=False, type=int, default='True') - p.add_argument('--aviation_processors', required=False, type=int, default='True') - p.add_argument('--point_sources_processors', required=False, type=int, default='True') - p.add_argument('--recreational_boats_processors', required=False, type=int, default='True') - p.add_argument('--shipping_port_processors', required=False, type=int, default='True') - p.add_argument('--residential_processors', required=False, type=int, default='True') - p.add_argument('--livestock_processors', required=False, type=int, default='True') - p.add_argument('--crop_operations_processors', required=False, type=int, default='True') - p.add_argument('--crop_fertilizers_processors', required=False, type=int, default='True') - p.add_argument('--agricultural_machinery_processors', required=False, type=int, default='True') + p.add_argument('--traffic_processors', required=True, type=int) + p.add_argument('--traffic_area_processors', required=True, type=int) + p.add_argument('--aviation_processors', required=True, type=int) + p.add_argument('--point_sources_processors', required=True, type=int) + p.add_argument('--recreational_boats_processors', required=True, type=int) + p.add_argument('--shipping_port_processors', required=True, type=int) + p.add_argument('--residential_processors', required=True, type=int) + p.add_argument('--livestock_processors', required=True, type=int) + p.add_argument('--crop_operations_processors', required=True, type=int) + p.add_argument('--crop_fertilizers_processors', required=True, type=int) + p.add_argument('--agricultural_machinery_processors', required=True, type=int) p.add_argument('--speciation_map', required=False, help='...') @@ -411,17 +399,17 @@ class Config(ArgParser): comm.Barrier() self.create_dir(arguments.auxiliary_files_path) - arguments.do_traffic = self._parse_bool(arguments.do_traffic) - arguments.do_traffic_area = self._parse_bool(arguments.do_traffic_area) - arguments.do_aviation = self._parse_bool(arguments.do_aviation) - arguments.do_point_sources = self._parse_bool(arguments.do_point_sources) - arguments.do_recreational_boats = self._parse_bool(arguments.do_recreational_boats) - arguments.do_shipping_port = self._parse_bool(arguments.do_shipping_port) - arguments.do_residential = self._parse_bool(arguments.do_residential) - arguments.do_livestock = self._parse_bool(arguments.do_livestock) - arguments.do_crop_operations = self._parse_bool(arguments.do_crop_operations) - arguments.do_crop_fertilizers = self._parse_bool(arguments.do_crop_fertilizers) - arguments.do_agricultural_machinery = self._parse_bool(arguments.do_agricultural_machinery) + arguments.do_traffic = arguments.traffic_processors > 0 + arguments.do_traffic_area = arguments.traffic_area_processors > 0 + arguments.do_aviation = arguments.aviation_processors > 0 + arguments.do_point_sources = arguments.point_sources_processors > 0 + arguments.do_recreational_boats = arguments.recreational_boats_processors > 0 + arguments.do_shipping_port = arguments.shipping_port_processors > 0 + arguments.do_residential = arguments.residential_processors > 0 + arguments.do_livestock = arguments.livestock_processors > 0 + arguments.do_crop_operations = arguments.crop_operations_processors > 0 + arguments.do_crop_fertilizers = arguments.crop_fertilizers_processors > 0 + arguments.do_agricultural_machinery = arguments.agricultural_machinery_processors > 0 # Aviation lists arguments.airport_list = self._parse_list(arguments.airport_list) diff --git a/hermesv3_bu/sectors/shipping_port_sector.py b/hermesv3_bu/sectors/shipping_port_sector.py index da880b63dacebea54ce9d009addd0aeed67374ba..bb6823bd1373cee9970276815873482073f4e4b3 100755 --- a/hermesv3_bu/sectors/shipping_port_sector.py +++ b/hermesv3_bu/sectors/shipping_port_sector.py @@ -6,6 +6,7 @@ import geopandas as gpd import numpy as np import timeit from hermesv3_bu.logger.log import Log +from hermesv3_bu.io_server.io_shapefile import IoShapefile class ShippingPortSector(Sector): @@ -81,9 +82,6 @@ class ShippingPortSector(Sector): spent_time = timeit.default_timer() logger.write_log('===== SHIPPING PORT SECTOR =====') - if comm.Get_size() > 1: - raise ValueError('Shipping port sector is not parallelised set shipping_port_processors to 1') - super(ShippingPortSector, self).__init__( comm, logger, auxiliary_dir, grid_shp, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -92,17 +90,12 @@ class ShippingPortSector(Sector): self.ef_engine = self.read_profiles(ef_dir) self.vessel_list = vessel_list - if port_list is not None: - unknown_ports = [x for x in port_list if x not in list(pd.read_csv(tonnage_path).code.values)] - if len(unknown_ports) > 0: - raise ValueError("The port(s) {0} are not listed on the {1} file.".format(unknown_ports, tonnage_path)) - self.port_list = port_list - else: - self.port_list = list(self.read_profiles(tonnage_path).code.values) self.hoteling_shapefile_path = hoteling_shapefile_path self.maneuvering_shapefile_path = maneuvering_shapefile_path + self.port_list = self.get_port_list() + self.engine_percent = self.read_profiles(engine_percent_path) self.tonnage = self.read_profiles(tonnage_path) self.tonnage.set_index('code', inplace=True) @@ -110,6 +103,28 @@ class ShippingPortSector(Sector): self.power_values = self.read_profiles(power_path) self.logger.write_time_log('ShippingPortSector', '__init__', timeit.default_timer() - spent_time) + def get_port_list(self): + if self.comm.Get_rank() == 0: + port_shp = IoShapefile(self.comm).read_shapefile_serial(self.maneuvering_shapefile_path) + port_shp.drop(columns=['Name', 'Weight'], inplace=True) + + port_shp = gpd.sjoin(port_shp, self.clip.shapefile.to_crs(port_shp.crs), how='inner', op='intersects') + port_list = np.unique(port_shp['code'].values) + print port_list + if len(port_list) < self.comm.Get_size(): + raise ValueError("The chosen number of processors {0} exceeds the number of involved ports {1}.".format( + self.comm.Get_size(), len(port_list)) + " Set {0} at shipping_port_processors value.".format( + len(port_list))) + port_list = np.array_split(port_list, self.comm.Get_size()) + else: + port_list = None + + port_list = self.comm.scatter(port_list, root=0) + + if len(port_list) == 0: + raise ValueError("The number ") + return list(port_list) + def read_monthly_profiles(self, path): """ Read the DataFrame of the monthly profiles with the month number as columns.