From 6668b28af9178678371050831d5989df4f8f3137 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 26 Aug 2019 11:23:11 +0200 Subject: [PATCH 1/6] - Traffic area: add logs to initialisation --- conf/hermes.conf | 6 +++--- hermesv3_bu/sectors/traffic_area_sector.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index a721eef..ca1c47c 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__crop_fert.nc +output_name = HERMESv3__tr_area.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -142,13 +142,13 @@ aviation_processors = 0 shipping_port_processors = 0 livestock_processors = 0 crop_operations_processors = 0 -crop_fertilizers_processors = 1 +crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 traffic_processors = 0 -traffic_area_processors = 0 +traffic_area_processors = 1 [SHAPEFILES] diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index f870161..7f537a3 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -32,6 +32,7 @@ class TrafficAreaSector(Sector): self.speciation_profiles_evaporative = self.read_speciation_profiles(speciation_profiles_evaporative) self.evaporative_ef_file = evaporative_ef_file if do_evaporative: + logger.write_log('\tInitialising evaporative emissions.', message_level=2) self.evaporative = self.init_evaporative(population_tif_path, nuts_shapefile, gasoline_path, total_pop_by_prov) else: @@ -44,6 +45,7 @@ class TrafficAreaSector(Sector): self.small_cities_weekly_profile = self.read_weekly_profiles(small_cities_weekly_profile) self.small_cities_hourly_profile = self.read_hourly_profiles(small_cities_hourly_profile) if do_small_cities: + logger.write_log('\tInitialising small cities emissions.', message_level=2) self.small_cities = self.init_small_cities(population_tif_path, small_cities_shp) else: self.small_cities = None @@ -55,18 +57,22 @@ class TrafficAreaSector(Sector): if self.comm.Get_rank() == 0: if not os.path.exists(os.path.join(self.auxiliary_dir, 'traffic_area', 'vehicle_by_cell.shp')): + self.logger.write_log('\t\tCreating population shapefile.', message_level=3) pop = self.get_clipped_population( global_path, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.shp')) + self.logger.write_log('\t\tCreating population shapefile by NUT.', message_level=3) pop = self.make_population_by_nuts( pop, provinces_shapefile, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_NUT.shp'), write_file=False) + self.logger.write_log('\t\tCreating population shapefile by NUT and cell.', message_level=3) pop = self.make_population_by_nuts_cell( pop, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_NUT_cell.shp')) - + self.logger.write_log('\t\tCreating vehicle shapefile by cell.', message_level=3) veh_cell = self.make_vehicles_by_cell( pop, gasoline_path, pd.read_csv(total_pop_by_prov), os.path.join(self.auxiliary_dir, 'traffic_area', 'vehicle_by_cell.shp')) else: + self.logger.write_log('\t\tReading vehicle shapefile by cell.', message_level=3) veh_cell = IoShapefile(self.comm).read_shapefile_serial( os.path.join(self.auxiliary_dir, 'traffic_area', 'vehicle_by_cell.shp')) else: @@ -81,14 +87,18 @@ class TrafficAreaSector(Sector): spent_time = timeit.default_timer() if self.comm.Get_rank() == 0: if not os.path.exists(os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL_cell.shp')): + self.logger.write_log('\t\tCreating population shapefile.', message_level=3) pop = self.get_clipped_population( global_path, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.shp')) + self.logger.write_log('\t\tCreating population small cities shapefile.', message_level=3) pop = self.make_population_by_nuts( pop, small_cities_shapefile, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL.shp'), - write_file=False) + write_file=True) + self.logger.write_log('\t\tCreating population small cities shapefile by cell.', message_level=3) pop = self.make_population_by_nuts_cell( pop, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL_cell.shp')) else: + self.logger.write_log('\t\tReading population small cities shapefile by cell.', message_level=3) pop = IoShapefile(self.comm).read_shapefile_serial( os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL_cell.shp')) else: -- GitLab From f11c28df9b62ca7d6c10e9b54e8be77bfc2ce81f Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 26 Aug 2019 11:25:34 +0200 Subject: [PATCH 2/6] - Traffic area: add logs --- hermesv3_bu/sectors/traffic_area_sector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 7f537a3..9f46a65 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -467,8 +467,10 @@ class TrafficAreaSector(Sector): spent_time = timeit.default_timer() if self.do_evaporative: + self.logger.write_log('\tCalculating evaporative emissions.', message_level=2) self.calculate_evaporative_emissions() if self.do_small_cities: + self.logger.write_log('\tCalculating small cities emissions.', message_level=2) self.calculate_small_cities_emissions() emissions = self.to_grid() -- GitLab From 66fb2d1d994059b5540035f24e31def79302d131 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 26 Aug 2019 11:27:29 +0200 Subject: [PATCH 3/6] -Aviation: corrected log message --- hermesv3_bu/sectors/aviation_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/aviation_sector.py b/hermesv3_bu/sectors/aviation_sector.py index 3bc15a0..ab7c605 100755 --- a/hermesv3_bu/sectors/aviation_sector.py +++ b/hermesv3_bu/sectors/aviation_sector.py @@ -380,7 +380,7 @@ class AviationSector(Sector): new_list = list(set(shp_airport_list) & set(airports_with_operations)) if len(new_list) != len(shp_airport_list): warn('{0} airports have no operations. Ignoring them.'.format( - list(set(new_list) - set(shp_airport_list)))) + list(set(shp_airport_list) - set(new_list)))) max_len = len(new_list) # Only for master (rank == 0) -- GitLab From 2c7c41488cecaa6f02c6eb1c12d596ee1a9b4383 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 26 Aug 2019 11:34:00 +0200 Subject: [PATCH 4/6] - Traffic area: To write all auxiliary files --- hermesv3_bu/sectors/traffic_area_sector.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 9f46a65..ddb974a 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -59,14 +59,14 @@ class TrafficAreaSector(Sector): if not os.path.exists(os.path.join(self.auxiliary_dir, 'traffic_area', 'vehicle_by_cell.shp')): self.logger.write_log('\t\tCreating population shapefile.', message_level=3) pop = self.get_clipped_population( - global_path, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.shp')) + global_path, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.shp'), write_file=True) self.logger.write_log('\t\tCreating population shapefile by NUT.', message_level=3) pop = self.make_population_by_nuts( pop, provinces_shapefile, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_NUT.shp'), - write_file=False) + write_file=True) self.logger.write_log('\t\tCreating population shapefile by NUT and cell.', message_level=3) pop = self.make_population_by_nuts_cell( - pop, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_NUT_cell.shp')) + pop, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_NUT_cell.shp'), write_file=True) self.logger.write_log('\t\tCreating vehicle shapefile by cell.', message_level=3) veh_cell = self.make_vehicles_by_cell( pop, gasoline_path, pd.read_csv(total_pop_by_prov), @@ -89,14 +89,14 @@ class TrafficAreaSector(Sector): if not os.path.exists(os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL_cell.shp')): self.logger.write_log('\t\tCreating population shapefile.', message_level=3) pop = self.get_clipped_population( - global_path, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.shp')) + global_path, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.shp'), write_file=True) self.logger.write_log('\t\tCreating population small cities shapefile.', message_level=3) pop = self.make_population_by_nuts( pop, small_cities_shapefile, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL.shp'), write_file=True) self.logger.write_log('\t\tCreating population small cities shapefile by cell.', message_level=3) pop = self.make_population_by_nuts_cell( - pop, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL_cell.shp')) + pop, os.path.join(self.auxiliary_dir, 'traffic_area', 'pop_SMALL_cell.shp'), write_file=True) else: self.logger.write_log('\t\tReading population small cities shapefile by cell.', message_level=3) pop = IoShapefile(self.comm).read_shapefile_serial( @@ -108,7 +108,7 @@ class TrafficAreaSector(Sector): self.logger.write_time_log('TrafficAreaSector', 'init_small_cities', timeit.default_timer() - spent_time) return pop - def get_clipped_population(self, global_path, population_shapefile_path): + def get_clipped_population(self, global_path, population_shapefile_path, write_file=True): from hermesv3_bu.io_server.io_raster import IoRaster spent_time = timeit.default_timer() @@ -116,7 +116,8 @@ class TrafficAreaSector(Sector): population_density = IoRaster(self.comm).clip_raster_with_shapefile_poly( global_path, self.clip.shapefile, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.tif')) - population_density = IoRaster(self.comm).to_shapefile_serie_by_cell(population_density) + population_density = IoRaster(self.comm).to_shapefile_serie_by_cell( + population_density, out_path=population_shapefile_path, write=write_file) else: population_density = IoShapefile(self.comm).read_shapefile_serial(population_shapefile_path) -- GitLab From 90505c84797542a220d3624bd206bda0850c6de3 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 26 Aug 2019 11:48:43 +0200 Subject: [PATCH 5/6] Modified log message --- hermesv3_bu/hermes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/hermes.py b/hermesv3_bu/hermes.py index d64e757..2dd8557 100755 --- a/hermesv3_bu/hermes.py +++ b/hermesv3_bu/hermes.py @@ -31,7 +31,7 @@ class Hermes(object): self.date_array = [self.arguments.start_date + timedelta(hours=hour) for hour in range(self.arguments.output_timestep_num)] self.logger.write_log('Dates to simulate: {0}'.format( - [aux_date.strftime("%Y/%m/%d, %H:%M:%S") for aux_date in self.date_array]), message_level=2) + [aux_date.strftime("\n\t%Y/%m/%d, %H:%M:%S") for aux_date in self.date_array]), message_level=3) self.sector_manager = SectorManager( self.comm, self.logger, self.grid, self.clip, self.date_array, self.arguments) -- GitLab From fb38653760e6897f465b3c55ff7649607ac4c720 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 26 Aug 2019 12:55:02 +0200 Subject: [PATCH 6/6] -Traffic area: Corrected parallel bug --- hermesv3_bu/sectors/traffic_area_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index ddb974a..a7cad42 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -275,7 +275,7 @@ class TrafficAreaSector(Sector): df1=self.evaporative, df2=temperature_mean, geom1_col='centroid', src_column='REC', axis=1) del self.evaporative['c_lat'], self.evaporative['c_lon'], self.evaporative['centroid'] - IoShapefile(self.comm).write_shapefile_serial( + IoShapefile(self.comm).write_shapefile_parallel( self.evaporative, os.path.join(self.auxiliary_dir, 'traffic_area', 'vehicle_by_cell.shp')) else: del self.evaporative['c_lat'], self.evaporative['c_lon'], self.evaporative['centroid'] -- GitLab