From f64c49efad6f0b7e4f8971ee97128b6aeae9cc43 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 2 Aug 2019 15:20:31 +0200 Subject: [PATCH 1/2] Traffic area working on Py3 --- conf/hermes.conf | 22 +++++++++++----------- hermesv3_bu/sectors/traffic_area_sector.py | 16 +++++++--------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index b1b8b16..8fffd69 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT/py3 -output_name = HERMES__py3_traffic_road.nc +output_name = HERMES__py3_traffic_area_cmaq.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -126,8 +126,8 @@ agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 -traffic_processors = 1 -traffic_area_processors = 0 +traffic_processors = 0 +traffic_area_processors = 1 [SHAPEFILES] @@ -136,7 +136,7 @@ nut_shapefile_ccaa = /Shapefiles/CCAA/ES_CCAA.shp population_density_map = /jrc/ghsl/original_files/GHS_POP_GPW42015_GLOBE_R2015A_54009_1k_v1_0.tif [SPECIATION DATA] -speciation_map = /profiles/speciation/map_cmaq_cb05_aero5.csv +speciation_map = /profiles/speciation/map_cmaq.csv molecular_weights = /profiles/speciation/MolecularWeights.csv [METEO PATHS] @@ -289,12 +289,12 @@ point_source_speciation_profiles = /profiles/speciation/point_sources point_source_measured_emissions = /point_sources/measured_emis/_measured_emissions.csv [TRAFFIC SECTOR] -do_hot = 0 -do_cold = 0 -do_tyre_wear = 0 -do_brake_wear = 0 +do_hot = 1 +do_cold = 1 +do_tyre_wear = 1 +do_brake_wear = 1 do_road_wear = 1 -do_resuspension = 0 +do_resuspension = 1 resuspension_correction = 1 write_rline = 0 @@ -323,11 +323,11 @@ traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 do_evaporative = 1 traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csv popullation_by_municipality = /traffic_area/population_by_mun.csv -traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_base.csv +traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_cmaq.csv traffic_area_evaporative_ef_file = /traffic_area/ef/evaporative_nmvoc.csv do_small_cities = 1 traffic_area_small_cities_path = /Shapefiles/small_cities/small_cities.shp -traffic_area_speciation_profiles_small_cities = /profiles/speciation/traffic_area/small_cities_base.csv +traffic_area_speciation_profiles_small_cities = /profiles/speciation/traffic_area/small_cities_cmaq.csv traffic_area_small_cities_ef_file = /traffic_area/ef/small_cities.csv small_cities_monthly_profile = /profiles/temporal/traffic_area/small_cities_monthly_profiles.csv small_cities_weekly_profile = /profiles/temporal/traffic_area/small_cities_weekly_profiles.csv diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 5a74458..aa2a32c 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -106,7 +106,7 @@ class TrafficAreaSector(Sector): population_density = IoRaster(self.comm).clip_raster_with_shapefile_poly( global_path, self.clip.shapefile, os.path.join(self.auxiliary_dir, 'traffic_area', 'population.tif')) - population_density = IoRaster(self.comm).to_shapefile_serie(population_density) + population_density = IoRaster(self.comm).to_shapefile_serie_by_cell(population_density) else: population_density = IoShapefile(self.comm).read_shapefile_serial(population_shapefile_path) @@ -170,10 +170,10 @@ class TrafficAreaSector(Sector): total_pop_by_nut.loc[:, column_id] = total_pop_by_nut[column_id].astype(np.int16) pop_nut_cell.loc[:, column_id] = pop_nut_cell[column_id].astype(np.int16) - df = pop_nut_cell.merge(total_pop_by_nut, left_on=column_id, right_on=column_id, how='left') + df = pd.merge(pop_nut_cell, total_pop_by_nut, left_on=column_id, right_on=column_id, how='left') df['pop_percent'] = df['data_x'] / df['data_y'] - del df['data_x'], df['data_y'], df['CELL_ID'] + df.drop(columns=['data_x', 'data_y'], inplace=True) gas_df = pd.read_csv(gasoline_path, index_col='COPERT_V_name').transpose() vehicle_type_list = list(gas_df.columns.values) @@ -361,7 +361,6 @@ class TrafficAreaSector(Sector): inc = 1 while len(grid.loc[grid['timezone'] == '', :]) > 0: - # print(len(grid.loc[grid['timezone'] == '', :])) grid.loc[grid['timezone'] == '', 'timezone'] = aux_grid.loc[grid['timezone'] == '', :].apply( lambda x: tz.closest_timezone_at(lng=x['lons'], lat=x['lats'], delta_degree=inc), axis=1) inc += 1 @@ -375,18 +374,17 @@ class TrafficAreaSector(Sector): p_names = small_cities.columns.values - aux_grid = self.grid_shp.loc[self.grid_shp['FID'].isin(small_cities.index.values), :] + aux_grid = self.grid_shp.loc[small_cities.index.values, :].reset_index().copy() aux_grid = self.add_timezone(aux_grid) aux_grid.set_index('FID', inplace=True) small_cities = small_cities.merge(aux_grid.loc[:, ['timezone']], left_index=True, right_index=True, how='left') - small_cities.loc[:, 'utc'] = self.date_array[0] + small_cities['utc'] = self.date_array[0] small_cities['date'] = small_cities.groupby('timezone')['utc'].apply( lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert(x.name).dt.tz_localize(None)) - small_cities.drop(['utc', 'timezone'], inplace=True, axis=1) - + small_cities.drop(columns=['utc', 'timezone'], inplace=True) df_list = [] for tstep in range(len(self.date_array)): small_cities['month'] = small_cities['date'].dt.month @@ -442,7 +440,7 @@ class TrafficAreaSector(Sector): spent_time = timeit.default_timer() if self.do_evaporative and self.do_small_cities: - dataset = pd.concat([self.evaporative, self.small_cities]) + dataset = pd.concat([self.evaporative, self.small_cities], sort=False) elif self.do_evaporative: dataset = self.evaporative elif self.do_small_cities: -- GitLab From 9c129d0800a09d629796f25819965e264d896f1d Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 2 Aug 2019 15:44:53 +0200 Subject: [PATCH 2/2] Drop warning on summary --- hermesv3_bu/writer/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/writer/writer.py b/hermesv3_bu/writer/writer.py index 06847f5..f1e9c2c 100755 --- a/hermesv3_bu/writer/writer.py +++ b/hermesv3_bu/writer/writer.py @@ -436,5 +436,5 @@ class Writer(object): summary.drop(columns=['layer'], inplace=True) summary.groupby('tstep').sum().to_csv(self.emission_summary_paths['hourly_summary_path']) summary.drop(columns=['tstep'], inplace=True) - summary.sum().to_csv(self.emission_summary_paths['total_summary_path']) + pd.DataFrame(summary.sum()).to_csv(self.emission_summary_paths['total_summary_path']) self.logger.write_time_log('Writer', 'make_summary', timeit.default_timer() - spent_time) -- GitLab