From 384bdf19046e0d3305d6f3c8059a0ce9b59c2c72 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 11:11:48 +0200 Subject: [PATCH 01/44] Traffic: Added traffic_scenario parameter --- conf/hermes.conf | 1 + hermesv3_bu/config/config.py | 2 ++ hermesv3_bu/sectors/sector_manager.py | 2 +- hermesv3_bu/sectors/traffic_sector.py | 5 +++-- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 189868e..37fd069 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -330,6 +330,7 @@ traffic_speciation_profile_tyre = /profiles/speciation/traffic/tyre_b traffic_speciation_profile_road = /profiles/speciation/traffic/road_base.csv traffic_speciation_profile_brake = /profiles/speciation/traffic/brake_base.csv traffic_speciation_profile_resuspension = /profiles/speciation/traffic/resuspension_base.csv +traffic_scenario = /traffic/shp_ZPE [TRAFFIC AREA SECTOR] traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index c3cde16..9e6e07a 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -627,6 +627,8 @@ class Config(ArgParser): p.add_argument('--traffic_speciation_profile_resuspension', required=False, help="Defines the path to the CSV file that contains the speciation profiles for the " + "resuspension emissions.") + p.add_argument('--traffic_scenario', required=False, default=None, + help="Defines the path to the shapefile that contains the traffic scenario by input pollutant.") # ***** TRAFFIC AREA SECTOR ***** p.add_argument('--traffic_area_pollutants', required=False, diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index b726d32..575730a 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -188,7 +188,7 @@ class SectorManager(object): arguments.temperature_hourly_files_path, arguments.output_dir, arguments.molecular_weights, arguments.resuspension_correction, arguments.precipitation_files_path, arguments.do_hot, arguments.do_cold, arguments.do_tyre_wear, arguments.do_brake_wear, arguments.do_road_wear, - arguments.do_resuspension, arguments.write_rline) + arguments.do_resuspension, arguments.write_rline, arguments.traffic_scenario) elif sector == 'traffic_area' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.traffic_area_sector import TrafficAreaSector diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 65a70a2..ac37728 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -17,7 +17,6 @@ from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.tools.checker import check_files, error_exit import gc -from memory_profiler import profile from ctypes import cdll, CDLL cdll.LoadLibrary("libc.so.6") libc = CDLL("libc.so.6") @@ -56,7 +55,7 @@ class TrafficSector(Sector): hot_cold_speciation=None, tyre_speciation=None, road_speciation=None, brake_speciation=None, resuspension_speciation=None, temp_common_path=None, output_dir=None, molecular_weights_path=None, resuspension_correction=True, precipitation_path=None, do_hot=True, do_cold=True, do_tyre_wear=True, - do_brake_wear=True, do_road_wear=True, do_resuspension=True, write_rline=False): + do_brake_wear=True, do_road_wear=True, do_resuspension=True, write_rline=False, traffic_scenario=None): spent_time = timeit.default_timer() logger.write_log('===== TRAFFIC SECTOR =====') @@ -137,6 +136,8 @@ class TrafficSector(Sector): self.check_profiles() self.expanded = self.expand_road_links() + self.traffic_scenario = traffic_scenario + del self.fleet_compo, self.speed_hourly, self.monthly_profiles, self.weekly_profiles, self.hourly_profiles libc.malloc_trim(0) gc.collect() -- GitLab From a9075fa189784f90c70b53d0c362e53c5ba0e17a Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 11:25:18 +0200 Subject: [PATCH 02/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index ac37728..05f83bb 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1420,6 +1420,11 @@ class TrafficSector(Sector): libc.malloc_trim(0) df_accum.set_index(['Link_ID', 'tstep'], inplace=True) + if self.traffic_scenario is not None: + + print(df_accum) + exit() + if self.write_rline: self.write_rline_output(df_accum.copy()) self.logger.write_log('\t\tRoad link emissions to grid.', message_level=2) -- GitLab From 33102e66e251fe0b48e8ac006ad5d815649a6c14 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 11:30:39 +0200 Subject: [PATCH 03/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 05f83bb..da27f7f 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -135,8 +135,13 @@ class TrafficSector(Sector): hourly_saturday_profiles_path, hourly_sunday_profiles_path) self.check_profiles() self.expanded = self.expand_road_links() - - self.traffic_scenario = traffic_scenario + self.traffic_scenario = None + if traffic_scenario is not None and not os.path.exists(traffic_scenario): + msg = "WARNING!!! " + msg += "Traffic scenario file '{0}' not found! Setting it to no scenario." + warn(msg) + else: + self.traffic_scenario = traffic_scenario del self.fleet_compo, self.speed_hourly, self.monthly_profiles, self.weekly_profiles, self.hourly_profiles libc.malloc_trim(0) @@ -1423,6 +1428,7 @@ class TrafficSector(Sector): if self.traffic_scenario is not None: print(df_accum) + print(df_accum.columns) exit() if self.write_rline: -- GitLab From 75be8c415dabebd258efd71e83856245e20d17b1 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 11:38:15 +0200 Subject: [PATCH 04/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index da27f7f..dd6e8c5 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1362,6 +1362,16 @@ class TrafficSector(Sector): self.logger.write_time_log('TrafficSector', 'speciate_traffic', timeit.default_timer() - spent_time) return df_out + def calculate_scenario(self, emissions): + if self.comm.Get_rank() == 0: + shp = IoShapefile(self.comm).read_shapefile_serial(self.traffic_scenario) + print(shp) + # print(emissions) + # print(emissions.columns) + self.comm.Barrier() + exit() + return emissions + def calculate_emissions(self): spent_time = timeit.default_timer() self.logger.write_log('\tCalculating Road traffic emissions', message_level=1) @@ -1426,10 +1436,7 @@ class TrafficSector(Sector): df_accum.set_index(['Link_ID', 'tstep'], inplace=True) if self.traffic_scenario is not None: - - print(df_accum) - print(df_accum.columns) - exit() + df_accum = self.calculate_scenario(df_accum) if self.write_rline: self.write_rline_output(df_accum.copy()) -- GitLab From 8c088bbd3cb0aead20d787c060e8c935357a988c Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 11:42:22 +0200 Subject: [PATCH 05/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index dd6e8c5..d073468 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1365,6 +1365,8 @@ class TrafficSector(Sector): def calculate_scenario(self, emissions): if self.comm.Get_rank() == 0: shp = IoShapefile(self.comm).read_shapefile_serial(self.traffic_scenario) + shp.rename(columns={'pm10': 'PM10'}, inplace=True) + IoShapefile(self.comm).write_shapefile_serial(shp, self.traffic_scenario) print(shp) # print(emissions) # print(emissions.columns) -- GitLab From ecd43ab2865bcad9c91689ae65b13d45e9d1e7b1 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 11:47:54 +0200 Subject: [PATCH 06/44] WIP --- hermesv3_bu/io_server/io_shapefile.py | 10 ++++++++++ hermesv3_bu/sectors/traffic_sector.py | 10 ++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/hermesv3_bu/io_server/io_shapefile.py b/hermesv3_bu/io_server/io_shapefile.py index 59ece64..2f1ddbc 100755 --- a/hermesv3_bu/io_server/io_shapefile.py +++ b/hermesv3_bu/io_server/io_shapefile.py @@ -88,6 +88,16 @@ class IoShapefile(IoServer): return data + def read_shapefile_broadcast(self, path, rank=0): + if self.comm.Get_rank() == rank: + data = self.read_shapefile_serial(path) + else: + data = None + + data = self.comm.bcast(data, root=0) + + return data + def split_shapefile(self, data, rank=0): """ diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index d073468..1ce883c 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1363,14 +1363,8 @@ class TrafficSector(Sector): return df_out def calculate_scenario(self, emissions): - if self.comm.Get_rank() == 0: - shp = IoShapefile(self.comm).read_shapefile_serial(self.traffic_scenario) - shp.rename(columns={'pm10': 'PM10'}, inplace=True) - IoShapefile(self.comm).write_shapefile_serial(shp, self.traffic_scenario) - print(shp) - # print(emissions) - # print(emissions.columns) - self.comm.Barrier() + scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.traffic_scenario) + print(scenario_shp) exit() return emissions -- GitLab From 88d687a98e640c17773a17932ffc53e06b2a1ebc Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:13:53 +0200 Subject: [PATCH 07/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 1ce883c..a79959e 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1362,8 +1362,10 @@ class TrafficSector(Sector): self.logger.write_time_log('TrafficSector', 'speciate_traffic', timeit.default_timer() - spent_time) return df_out - def calculate_scenario(self, emissions): + def apply_scenario(self, emissions): + self.logger.write_log('\t\tApplying emission scenario', message_level=2) scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.traffic_scenario) + scenario_shp = gpd.sjoin(emissions, scenario_shp, how='left', rsuffix='_f', lsuffix=None) print(scenario_shp) exit() return emissions @@ -1432,7 +1434,7 @@ class TrafficSector(Sector): df_accum.set_index(['Link_ID', 'tstep'], inplace=True) if self.traffic_scenario is not None: - df_accum = self.calculate_scenario(df_accum) + df_accum = self.apply_scenario(df_accum) if self.write_rline: self.write_rline_output(df_accum.copy()) -- GitLab From ddd202c6f7956ce8f8b56ce3729edd9e0e313b72 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:16:46 +0200 Subject: [PATCH 08/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index a79959e..8f98be9 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1365,8 +1365,9 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.traffic_scenario) - scenario_shp = gpd.sjoin(emissions, scenario_shp, how='left', rsuffix='_f', lsuffix=None) - print(scenario_shp) + emisisons = gpd.sjoin(emissions, scenario_shp, how='left', rsuffix='_f', lsuffix=None) + print(emisisons) + print(emisisons.columns) exit() return emissions -- GitLab From a68cb29db61e16a1f40a3640b08614d8971e6047 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:19:09 +0200 Subject: [PATCH 09/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 8f98be9..34a02e2 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1365,7 +1365,7 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.traffic_scenario) - emisisons = gpd.sjoin(emissions, scenario_shp, how='left', rsuffix='_f', lsuffix=None) + emisisons = gpd.sjoin(emissions, scenario_shp, how='left', rsuffix='_f') print(emisisons) print(emisisons.columns) exit() -- GitLab From 886d3f3b328fe576b6e6632f7baa5c2929f3391f Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:26:42 +0200 Subject: [PATCH 10/44] WIP --- hermesv3_bu/io_server/io_shapefile.py | 12 ++++++++++++ hermesv3_bu/sectors/traffic_sector.py | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/hermesv3_bu/io_server/io_shapefile.py b/hermesv3_bu/io_server/io_shapefile.py index 2f1ddbc..a1aa341 100755 --- a/hermesv3_bu/io_server/io_shapefile.py +++ b/hermesv3_bu/io_server/io_shapefile.py @@ -89,6 +89,18 @@ class IoShapefile(IoServer): return data def read_shapefile_broadcast(self, path, rank=0): + """ + The master process reads the shapefile and broadcast it. + + :param path: Path to the shapefile + :type path: str + + :param rank: Master rank + :type rank: int + + :return: Shapefile + :rtype: GeoDataFrame + """ if self.comm.Get_rank() == rank: data = self.read_shapefile_serial(path) else: diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 34a02e2..d7f6540 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1365,7 +1365,9 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.traffic_scenario) - emisisons = gpd.sjoin(emissions, scenario_shp, how='left', rsuffix='_f') + for col_name in scenario_shp.columns: + scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) + emisisons = gpd.sjoin(emissions, scenario_shp, how='left') print(emisisons) print(emisisons.columns) exit() -- GitLab From e55f3e48d34020196bbebd9f78b7b616ba32b2ee Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:32:42 +0200 Subject: [PATCH 11/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index d7f6540..dd20436 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1364,12 +1364,19 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) + print(emissions) + print(emissions.columns) scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.traffic_scenario) for col_name in scenario_shp.columns: scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) - emisisons = gpd.sjoin(emissions, scenario_shp, how='left') - print(emisisons) - print(emisisons.columns) + print('SCENARIO') + print(scenario_shp) + print(scenario_shp.columns) + emissions = gpd.sjoin(emissions, scenario_shp, how='left') + print('EMISSIONS') + print(emissions) + print(emissions.columns) + self.logger.write_log('{0}'.format(emissions.columns.to_list()), message_level=3) exit() return emissions -- GitLab From 040014afd058964da37fdbfe6d2b9f48b8e64105 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:40:07 +0200 Subject: [PATCH 12/44] WIP --- hermesv3_bu/sectors/sector_manager.py | 2 +- hermesv3_bu/sectors/traffic_sector.py | 35 ++++++++++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index 575730a..d7d9797 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -188,7 +188,7 @@ class SectorManager(object): arguments.temperature_hourly_files_path, arguments.output_dir, arguments.molecular_weights, arguments.resuspension_correction, arguments.precipitation_files_path, arguments.do_hot, arguments.do_cold, arguments.do_tyre_wear, arguments.do_brake_wear, arguments.do_road_wear, - arguments.do_resuspension, arguments.write_rline, arguments.traffic_scenario) + arguments.do_resuspension, arguments.write_rline, arguments.scenario) elif sector == 'traffic_area' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.traffic_area_sector import TrafficAreaSector diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index dd20436..4630034 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -135,13 +135,8 @@ class TrafficSector(Sector): hourly_saturday_profiles_path, hourly_sunday_profiles_path) self.check_profiles() self.expanded = self.expand_road_links() - self.traffic_scenario = None - if traffic_scenario is not None and not os.path.exists(traffic_scenario): - msg = "WARNING!!! " - msg += "Traffic scenario file '{0}' not found! Setting it to no scenario." - warn(msg) - else: - self.traffic_scenario = traffic_scenario + + self.scenario = self.get_scenario(traffic_scenario) del self.fleet_compo, self.speed_hourly, self.monthly_profiles, self.weekly_profiles, self.hourly_profiles libc.malloc_trim(0) @@ -173,6 +168,25 @@ class TrafficSector(Sector): libc.malloc_trim(0) gc.collect() + def get_scenario(self, scenario_path): + if scenario_path is not None and not os.path.exists(scenario_path): + msg = "WARNING!!! " + msg += "Traffic scenario file '{0}' not found! Setting it to no scenario." + warn(msg) + scenario = None + else: + # scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.scenario) + # for col_name in scenario_shp.columns: + # scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) + # print('SCENARIO') + # print(scenario_shp) + # print(scenario_shp.columns) + # emissions = gpd.sjoin(self.road_links, scenario_shp, how='left') + + scenario = scenario_path + + return scenario + def check_profiles(self): spent_time = timeit.default_timer() # Checking speed profiles IDs @@ -1366,7 +1380,10 @@ class TrafficSector(Sector): self.logger.write_log('\t\tApplying emission scenario', message_level=2) print(emissions) print(emissions.columns) - scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.traffic_scenario) + print('ROAD LINKS') + print(self.road_links) + print(self.road_links.columns) + scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.scenario) for col_name in scenario_shp.columns: scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) print('SCENARIO') @@ -1443,7 +1460,7 @@ class TrafficSector(Sector): libc.malloc_trim(0) df_accum.set_index(['Link_ID', 'tstep'], inplace=True) - if self.traffic_scenario is not None: + if self.scenario is not None: df_accum = self.apply_scenario(df_accum) if self.write_rline: -- GitLab From fb8bc1b26119617b4dc47e9504fc230fe070a7ea Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:44:13 +0200 Subject: [PATCH 13/44] WIP --- hermesv3_bu/sectors/sector_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index d7d9797..575730a 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -188,7 +188,7 @@ class SectorManager(object): arguments.temperature_hourly_files_path, arguments.output_dir, arguments.molecular_weights, arguments.resuspension_correction, arguments.precipitation_files_path, arguments.do_hot, arguments.do_cold, arguments.do_tyre_wear, arguments.do_brake_wear, arguments.do_road_wear, - arguments.do_resuspension, arguments.write_rline, arguments.scenario) + arguments.do_resuspension, arguments.write_rline, arguments.traffic_scenario) elif sector == 'traffic_area' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.traffic_area_sector import TrafficAreaSector -- GitLab From a2f8a455ef0fd0358ff6bda0b94454dd3eac427d Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:48:04 +0200 Subject: [PATCH 14/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 4630034..c10f252 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -175,13 +175,17 @@ class TrafficSector(Sector): warn(msg) scenario = None else: - # scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.scenario) - # for col_name in scenario_shp.columns: - # scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) - # print('SCENARIO') - # print(scenario_shp) - # print(scenario_shp.columns) - # emissions = gpd.sjoin(self.road_links, scenario_shp, how='left') + scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.scenario) + for col_name in scenario_shp.columns: + scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) + print('SCENARIO SHP') + print(scenario_shp) + print(scenario_shp.columns) + scenario = gpd.sjoin(self.road_links, scenario_shp, how='left') + print('SCENARIO') + print(scenario) + print(scenario.columns) + exit() scenario = scenario_path -- GitLab From 4b6d84b6b4e003b61154a475abaa01f8087a326e Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:50:39 +0200 Subject: [PATCH 15/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index c10f252..055ec8e 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -175,7 +175,7 @@ class TrafficSector(Sector): warn(msg) scenario = None else: - scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.scenario) + scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(scenario_path) for col_name in scenario_shp.columns: scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) print('SCENARIO SHP') -- GitLab From 26555087290782947904a57f7ec1e730558cd4ce Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:53:45 +0200 Subject: [PATCH 16/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 055ec8e..ae697cd 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -176,12 +176,10 @@ class TrafficSector(Sector): scenario = None else: scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(scenario_path) - for col_name in scenario_shp.columns: - scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) print('SCENARIO SHP') print(scenario_shp) print(scenario_shp.columns) - scenario = gpd.sjoin(self.road_links, scenario_shp, how='left') + scenario = gpd.sjoin(self.road_links, scenario_shp, how='left', lsuffix='_f') print('SCENARIO') print(scenario) print(scenario.columns) -- GitLab From eef9ae3b00a223f45f493eb2597773733b35983a Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 12:57:57 +0200 Subject: [PATCH 17/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index ae697cd..50dfcf1 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -173,19 +173,21 @@ class TrafficSector(Sector): msg = "WARNING!!! " msg += "Traffic scenario file '{0}' not found! Setting it to no scenario." warn(msg) - scenario = None - else: + scenario_path = None + if scenario_path is not None: scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(scenario_path) print('SCENARIO SHP') print(scenario_shp) print(scenario_shp.columns) - scenario = gpd.sjoin(self.road_links, scenario_shp, how='left', lsuffix='_f') + scenario = gpd.sjoin(self.road_links['geometry'], scenario_shp, how='left') print('SCENARIO') print(scenario) print(scenario.columns) exit() scenario = scenario_path + else: + scenario = None return scenario -- GitLab From d96856954531ded218d02d3971c2e48f1e1b560f Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 13:00:27 +0200 Subject: [PATCH 18/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 50dfcf1..bd7d11d 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -179,7 +179,7 @@ class TrafficSector(Sector): print('SCENARIO SHP') print(scenario_shp) print(scenario_shp.columns) - scenario = gpd.sjoin(self.road_links['geometry'], scenario_shp, how='left') + scenario = gpd.sjoin(self.road_links[['geometry']], scenario_shp, how='left') print('SCENARIO') print(scenario) print(scenario.columns) -- GitLab From 8cc7d6b125d79360b8dda592c837cfefbc8748bc Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 13:21:06 +0200 Subject: [PATCH 19/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index bd7d11d..9792cdf 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -174,15 +174,19 @@ class TrafficSector(Sector): msg += "Traffic scenario file '{0}' not found! Setting it to no scenario." warn(msg) scenario_path = None + if scenario_path is not None: + self.logger.write_log('\t\tGetting emission scenario', message_level=2) scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(scenario_path) print('SCENARIO SHP') print(scenario_shp) print(scenario_shp.columns) - scenario = gpd.sjoin(self.road_links[['geometry']], scenario_shp, how='left') + scenario = gpd.sjoin(GeoDataFrame(index=self.road_links.index, geometry=self.road_links.geometry.centroid, + crs=self.road_links.crs), scenario_shp, how='left') print('SCENARIO') print(scenario) print(scenario.columns) + print(scenario.loc[~scenario['PM10'].isna()]) exit() scenario = scenario_path -- GitLab From 7a6b0402bf34243a9639fd65381a9db03c957880 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 13:25:48 +0200 Subject: [PATCH 20/44] WIP --- hermesv3_bu/io_server/io_shapefile.py | 7 ++++++- hermesv3_bu/sectors/traffic_sector.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/io_server/io_shapefile.py b/hermesv3_bu/io_server/io_shapefile.py index a1aa341..bdc397f 100755 --- a/hermesv3_bu/io_server/io_shapefile.py +++ b/hermesv3_bu/io_server/io_shapefile.py @@ -88,7 +88,7 @@ class IoShapefile(IoServer): return data - def read_shapefile_broadcast(self, path, rank=0): + def read_shapefile_broadcast(self, path, rank=0, crs=None): """ The master process reads the shapefile and broadcast it. @@ -98,11 +98,16 @@ class IoShapefile(IoServer): :param rank: Master rank :type rank: int + :param crs: Projection desired. + :type crs: None, str + :return: Shapefile :rtype: GeoDataFrame """ if self.comm.Get_rank() == rank: data = self.read_shapefile_serial(path) + if crs is not None: + data = data.to_crs(crs) else: data = None diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 9792cdf..d026408 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -177,7 +177,7 @@ class TrafficSector(Sector): if scenario_path is not None: self.logger.write_log('\t\tGetting emission scenario', message_level=2) - scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(scenario_path) + scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(scenario_path, crs=self.road_links.crs) print('SCENARIO SHP') print(scenario_shp) print(scenario_shp.columns) -- GitLab From 15b93c72dfe880d97ab331c556bb51b6b401e60c Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 13:30:05 +0200 Subject: [PATCH 21/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index d026408..e37316b 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -186,7 +186,9 @@ class TrafficSector(Sector): print('SCENARIO') print(scenario) print(scenario.columns) - print(scenario.loc[~scenario['PM10'].isna()]) + if len(scenario.loc[~scenario['PM10'].isna()]) > 0: + print(scenario.loc[~scenario['PM10'].isna()]) + print("{0} ".format(self.comm.Get_rank()) * 100) exit() scenario = scenario_path -- GitLab From 663a979ba3a41c44c0b6b71bd425f37344e01d8d Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 13:56:31 +0200 Subject: [PATCH 22/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 35 ++++++--------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index e37316b..18fa94c 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -178,20 +178,12 @@ class TrafficSector(Sector): if scenario_path is not None: self.logger.write_log('\t\tGetting emission scenario', message_level=2) scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(scenario_path, crs=self.road_links.crs) - print('SCENARIO SHP') - print(scenario_shp) - print(scenario_shp.columns) + scenario = gpd.sjoin(GeoDataFrame(index=self.road_links.index, geometry=self.road_links.geometry.centroid, crs=self.road_links.crs), scenario_shp, how='left') - print('SCENARIO') - print(scenario) - print(scenario.columns) - if len(scenario.loc[~scenario['PM10'].isna()]) > 0: - print(scenario.loc[~scenario['PM10'].isna()]) - print("{0} ".format(self.comm.Get_rank()) * 100) - exit() - - scenario = scenario_path + scenario = scenario.loc[:, scenario_shp.columns] + scenario.fillna(1, inplace=True) + else: scenario = None @@ -1388,22 +1380,9 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) - print(emissions) - print(emissions.columns) - print('ROAD LINKS') - print(self.road_links) - print(self.road_links.columns) - scenario_shp = IoShapefile(self.comm).read_shapefile_broadcast(self.scenario) - for col_name in scenario_shp.columns: - scenario_shp.rename(columns={col_name: '{0}_f'.format(col_name)}, inplace=True) - print('SCENARIO') - print(scenario_shp) - print(scenario_shp.columns) - emissions = gpd.sjoin(emissions, scenario_shp, how='left') - print('EMISSIONS') - print(emissions) - print(emissions.columns) - self.logger.write_log('{0}'.format(emissions.columns.to_list()), message_level=3) + emis_aux = emissions.join(self.scenario, rsuffix='_f') + print(emis_aux) + print(emis_aux.columns) exit() return emissions -- GitLab From 36190a5cd060e16b29a4afdce1d9c59e13f929ce Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 13:59:58 +0200 Subject: [PATCH 23/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 18fa94c..ee38f4c 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1380,6 +1380,12 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) + print('SCENARIO') + print(self.scenario) + print(self.scenario.colums) + print('EMISSIONS') + print(emissions) + print(emissions.columns) emis_aux = emissions.join(self.scenario, rsuffix='_f') print(emis_aux) print(emis_aux.columns) -- GitLab From 19f621c729f2f83a229a09d8016609c436b0bde6 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 14:08:19 +0200 Subject: [PATCH 24/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index ee38f4c..47ac8b4 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1382,7 +1382,7 @@ class TrafficSector(Sector): self.logger.write_log('\t\tApplying emission scenario', message_level=2) print('SCENARIO') print(self.scenario) - print(self.scenario.colums) + print(self.scenario.columns) print('EMISSIONS') print(emissions) print(emissions.columns) -- GitLab From 499852aa5aeb50e539ba0acf89aed8e459e97340 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 14:09:09 +0200 Subject: [PATCH 25/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 47ac8b4..dd52279 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -182,6 +182,8 @@ class TrafficSector(Sector): scenario = gpd.sjoin(GeoDataFrame(index=self.road_links.index, geometry=self.road_links.geometry.centroid, crs=self.road_links.crs), scenario_shp, how='left') scenario = scenario.loc[:, scenario_shp.columns] + print(scenario) + exit() scenario.fillna(1, inplace=True) else: -- GitLab From 04a25508cdd724583e0a6685d018dd31b37eeb7a Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 14:12:01 +0200 Subject: [PATCH 26/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index dd52279..2515c1a 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -182,8 +182,8 @@ class TrafficSector(Sector): scenario = gpd.sjoin(GeoDataFrame(index=self.road_links.index, geometry=self.road_links.geometry.centroid, crs=self.road_links.crs), scenario_shp, how='left') scenario = scenario.loc[:, scenario_shp.columns] + scenario.index.name=['Link_ID'] print(scenario) - exit() scenario.fillna(1, inplace=True) else: -- GitLab From 29a702fcd26f4be2c96d4ab7481e9b4596f7f95a Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 14:14:06 +0200 Subject: [PATCH 27/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 2515c1a..58bedec 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -182,7 +182,7 @@ class TrafficSector(Sector): scenario = gpd.sjoin(GeoDataFrame(index=self.road_links.index, geometry=self.road_links.geometry.centroid, crs=self.road_links.crs), scenario_shp, how='left') scenario = scenario.loc[:, scenario_shp.columns] - scenario.index.name=['Link_ID'] + scenario.index.name = ['Link_ID'] print(scenario) scenario.fillna(1, inplace=True) -- GitLab From cb9ce5adf07696aa4d2c7f51cf2aaeed8f03b2f6 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 14:15:55 +0200 Subject: [PATCH 28/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 58bedec..06a0ad0 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -182,7 +182,7 @@ class TrafficSector(Sector): scenario = gpd.sjoin(GeoDataFrame(index=self.road_links.index, geometry=self.road_links.geometry.centroid, crs=self.road_links.crs), scenario_shp, how='left') scenario = scenario.loc[:, scenario_shp.columns] - scenario.index.name = ['Link_ID'] + scenario.index.name = 'Link_ID' print(scenario) scenario.fillna(1, inplace=True) -- GitLab From a231faea02ac79262aefc05949299e2121137469 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 14:21:27 +0200 Subject: [PATCH 29/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 06a0ad0..1d305c4 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -182,8 +182,8 @@ class TrafficSector(Sector): scenario = gpd.sjoin(GeoDataFrame(index=self.road_links.index, geometry=self.road_links.geometry.centroid, crs=self.road_links.crs), scenario_shp, how='left') scenario = scenario.loc[:, scenario_shp.columns] - scenario.index.name = 'Link_ID' - print(scenario) + scenario = pd.DataFrame(scenario.drop(columns='geometry'), index=pd.Index(scenario.index, name='Link_ID')) + scenario.fillna(1, inplace=True) else: -- GitLab From f3be77bbbec82e0bd1ca83d4bd43e013e0170322 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 15:10:30 +0200 Subject: [PATCH 30/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 1d305c4..a385ef9 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1389,9 +1389,10 @@ class TrafficSector(Sector): print(emissions) print(emissions.columns) emis_aux = emissions.join(self.scenario, rsuffix='_f') - print(emis_aux) - print(emis_aux.columns) - exit() + for pollutant in self.scenario.columns: + if pollutant in emissions.columns: + self.logger.write_log('\t\t\tApplying emission scenario for {0}'.format(pollutant), message_level=3) + emissions.loc[emis_aux.index, pollutant] *= emis_aux.loc[:, '{0}_f'.format(pollutant)] return emissions def calculate_emissions(self): -- GitLab From 55099b8eedb8b1298cb7d7428f4b5d90cc22325f Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 3 Sep 2021 15:23:28 +0200 Subject: [PATCH 31/44] Finished implementation --- hermesv3_bu/sectors/traffic_sector.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index a385ef9..4b00042 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1382,12 +1382,6 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) - print('SCENARIO') - print(self.scenario) - print(self.scenario.columns) - print('EMISSIONS') - print(emissions) - print(emissions.columns) emis_aux = emissions.join(self.scenario, rsuffix='_f') for pollutant in self.scenario.columns: if pollutant in emissions.columns: -- GitLab From c46f122c0b9c994565290b867bbb97db1619f44c Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 6 Oct 2021 11:21:59 +0200 Subject: [PATCH 32/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 4b00042..2584fac 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1382,7 +1382,12 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) - emis_aux = emissions.join(self.scenario, rsuffix='_f') + try: + emis_aux = emissions.join(self.scenario, rsuffix='_f') + except NotImplementedError as e: + print("Rank {0} Emis: {1}".format(self.comm.Get_rank(), emissions)) + sys.stdout.flush() + error_exit(e) for pollutant in self.scenario.columns: if pollutant in emissions.columns: self.logger.write_log('\t\t\tApplying emission scenario for {0}'.format(pollutant), message_level=3) -- GitLab From c82adf4a240cc426a255b2281672e9aff2cddb65 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 6 Oct 2021 11:31:13 +0200 Subject: [PATCH 33/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 2584fac..7b7e5bb 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1386,6 +1386,7 @@ class TrafficSector(Sector): emis_aux = emissions.join(self.scenario, rsuffix='_f') except NotImplementedError as e: print("Rank {0} Emis: {1}".format(self.comm.Get_rank(), emissions)) + emissions.to_file("/gpfs/scratch/bsc32/bsc32854/hermes_test_escenarios/hermes_test_escenarios_OUT/shp_err/r_{0}".format(self.comm.Get_rank())) sys.stdout.flush() error_exit(e) for pollutant in self.scenario.columns: -- GitLab From d4231d1c449f71722c59b6beea396e414e4aa1f5 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 6 Oct 2021 11:43:57 +0200 Subject: [PATCH 34/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 7b7e5bb..09c0066 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1383,7 +1383,7 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) try: - emis_aux = emissions.join(self.scenario, rsuffix='_f') + emis_aux = emissions.sjoin(self.scenario, rsuffix='_f') except NotImplementedError as e: print("Rank {0} Emis: {1}".format(self.comm.Get_rank(), emissions)) emissions.to_file("/gpfs/scratch/bsc32/bsc32854/hermes_test_escenarios/hermes_test_escenarios_OUT/shp_err/r_{0}".format(self.comm.Get_rank())) -- GitLab From 71d5c19ce9ccbf59b7a02f80cc40efebcb7aae89 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 6 Oct 2021 11:49:16 +0200 Subject: [PATCH 35/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 09c0066..e25ae44 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1383,9 +1383,9 @@ class TrafficSector(Sector): def apply_scenario(self, emissions): self.logger.write_log('\t\tApplying emission scenario', message_level=2) try: - emis_aux = emissions.sjoin(self.scenario, rsuffix='_f') + emis_aux = emissions.join(self.scenario, rsuffix='_f') except NotImplementedError as e: - print("Rank {0} Emis: {1}".format(self.comm.Get_rank(), emissions)) + print("Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) emissions.to_file("/gpfs/scratch/bsc32/bsc32854/hermes_test_escenarios/hermes_test_escenarios_OUT/shp_err/r_{0}".format(self.comm.Get_rank())) sys.stdout.flush() error_exit(e) -- GitLab From e1d1af4e6784fd1229d4024432cc5d8d0a6984d6 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 6 Oct 2021 11:57:35 +0200 Subject: [PATCH 36/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index e25ae44..4f8dfd1 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1384,8 +1384,10 @@ class TrafficSector(Sector): self.logger.write_log('\t\tApplying emission scenario', message_level=2) try: emis_aux = emissions.join(self.scenario, rsuffix='_f') + print("OK: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) + except NotImplementedError as e: - print("Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) + print("ERROR: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) emissions.to_file("/gpfs/scratch/bsc32/bsc32854/hermes_test_escenarios/hermes_test_escenarios_OUT/shp_err/r_{0}".format(self.comm.Get_rank())) sys.stdout.flush() error_exit(e) -- GitLab From b1bb3339646de23c9bc9e83aa0adaa0d11ce6d75 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 6 Oct 2021 12:02:34 +0200 Subject: [PATCH 37/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 4f8dfd1..eb1e5b9 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1385,7 +1385,7 @@ class TrafficSector(Sector): try: emis_aux = emissions.join(self.scenario, rsuffix='_f') print("OK: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) - + sys.stdout.flush() except NotImplementedError as e: print("ERROR: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) emissions.to_file("/gpfs/scratch/bsc32/bsc32854/hermes_test_escenarios/hermes_test_escenarios_OUT/shp_err/r_{0}".format(self.comm.Get_rank())) -- GitLab From 7412ab1182aabebd2e3e592ff7ec408d98a83175 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 6 Oct 2021 12:11:00 +0200 Subject: [PATCH 38/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index eb1e5b9..83b42a7 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1390,7 +1390,7 @@ class TrafficSector(Sector): print("ERROR: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) emissions.to_file("/gpfs/scratch/bsc32/bsc32854/hermes_test_escenarios/hermes_test_escenarios_OUT/shp_err/r_{0}".format(self.comm.Get_rank())) sys.stdout.flush() - error_exit(e) + raise e for pollutant in self.scenario.columns: if pollutant in emissions.columns: self.logger.write_log('\t\t\tApplying emission scenario for {0}'.format(pollutant), message_level=3) -- GitLab From 343cca967c254e1ca62a775234170a0d4ac9525e Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 20 Oct 2021 11:50:46 +0200 Subject: [PATCH 39/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 83b42a7..91ecf04 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1388,7 +1388,7 @@ class TrafficSector(Sector): sys.stdout.flush() except NotImplementedError as e: print("ERROR: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) - emissions.to_file("/gpfs/scratch/bsc32/bsc32854/hermes_test_escenarios/hermes_test_escenarios_OUT/shp_err/r_{0}".format(self.comm.Get_rank())) + emissions.to_file("/gpfs/scratch/bsc32/bsc32538/TraffisScenarios/OUT/aux/r_{0}".format(self.comm.Get_rank())) sys.stdout.flush() raise e for pollutant in self.scenario.columns: -- GitLab From 605048449b703253d06faa9f95001e01a312268c Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 20 Oct 2021 12:04:03 +0200 Subject: [PATCH 40/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 91ecf04..6b97346 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1381,6 +1381,8 @@ class TrafficSector(Sector): return df_out def apply_scenario(self, emissions): + emissions.index = emissions.index.set_levels(emissions.index.levels['Link_ID'].astype(int), level='Link_ID') + self.logger.write_log('\t\tApplying emission scenario', message_level=2) try: emis_aux = emissions.join(self.scenario, rsuffix='_f') -- GitLab From 8a7fb6ca36f2f73eb75310d80684054f130b1859 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 20 Oct 2021 12:09:34 +0200 Subject: [PATCH 41/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 6b97346..aeeecbc 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1381,11 +1381,11 @@ class TrafficSector(Sector): return df_out def apply_scenario(self, emissions): - emissions.index = emissions.index.set_levels(emissions.index.levels['Link_ID'].astype(int), level='Link_ID') + emissions.index = emissions.index.set_levels(emissions.index.levels[0].astype(int), level=0) self.logger.write_log('\t\tApplying emission scenario', message_level=2) try: - emis_aux = emissions.join(self.scenario, rsuffix='_f') + emis_aux = emissions.join(self.scenario, on='Link_ID', rsuffix='_f') print("OK: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) sys.stdout.flush() except NotImplementedError as e: -- GitLab From 426c403c537bcf5b61965b3a1b6a255647b47997 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 20 Oct 2021 12:13:36 +0200 Subject: [PATCH 42/44] WIP --- hermesv3_bu/sectors/traffic_sector.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index aeeecbc..5f167c6 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -1384,15 +1384,8 @@ class TrafficSector(Sector): emissions.index = emissions.index.set_levels(emissions.index.levels[0].astype(int), level=0) self.logger.write_log('\t\tApplying emission scenario', message_level=2) - try: - emis_aux = emissions.join(self.scenario, on='Link_ID', rsuffix='_f') - print("OK: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) - sys.stdout.flush() - except NotImplementedError as e: - print("ERROR: Rank {0} Emis: {1}\n Scenario: {2}".format(self.comm.Get_rank(), emissions, self.scenario)) - emissions.to_file("/gpfs/scratch/bsc32/bsc32538/TraffisScenarios/OUT/aux/r_{0}".format(self.comm.Get_rank())) - sys.stdout.flush() - raise e + emis_aux = emissions.join(self.scenario, on='Link_ID', rsuffix='_f') + for pollutant in self.scenario.columns: if pollutant in emissions.columns: self.logger.write_log('\t\t\tApplying emission scenario for {0}'.format(pollutant), message_level=3) -- GitLab From 260f483aa02cc5bd0e541798113aeeb2a9295317 Mon Sep 17 00:00:00 2001 From: ctena Date: Thu, 23 Dec 2021 09:09:08 +0100 Subject: [PATCH 43/44] ERROR message when the traffic scenario is set but it doesn't exist --- hermesv3_bu/sectors/traffic_sector.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 5f167c6..6f18103 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -135,7 +135,8 @@ class TrafficSector(Sector): hourly_saturday_profiles_path, hourly_sunday_profiles_path) self.check_profiles() self.expanded = self.expand_road_links() - + if traffic_scenario in ['=', '', ' ', 'None']: + traffic_scenario = None self.scenario = self.get_scenario(traffic_scenario) del self.fleet_compo, self.speed_hourly, self.monthly_profiles, self.weekly_profiles, self.hourly_profiles @@ -170,10 +171,9 @@ class TrafficSector(Sector): def get_scenario(self, scenario_path): if scenario_path is not None and not os.path.exists(scenario_path): - msg = "WARNING!!! " - msg += "Traffic scenario file '{0}' not found! Setting it to no scenario." - warn(msg) - scenario_path = None + msg = "ERROR!!! " + msg += "Traffic scenario file '{0}' not found!".format(scenario_path) + error_exit(msg) if scenario_path is not None: self.logger.write_log('\t\tGetting emission scenario', message_level=2) -- GitLab From 7b2beb044876f7b4e3edacf2303d1a477e191fd6 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 12 Jan 2022 11:18:23 +0100 Subject: [PATCH 44/44] ERROR message when the traffic scenario is set but it doesn't exist --- hermesv3_bu/sectors/traffic_sector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 6f18103..f2060d5 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -170,6 +170,8 @@ class TrafficSector(Sector): gc.collect() def get_scenario(self, scenario_path): + if scenario_path in ['', 'None']: + scenario_path = None if scenario_path is not None and not os.path.exists(scenario_path): msg = "ERROR!!! " msg += "Traffic scenario file '{0}' not found!".format(scenario_path) -- GitLab