diff --git a/conf/hermes.conf b/conf/hermes.conf index 5ebfb165169dac65d17597ca33e7139353ed7334..a721eef94b1e83a548a6d13874050ca28ab443ee 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,14 +3,14 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__point_new.nc -emission_summary = 1 +output_name = HERMESv3__crop_fert.nc +emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 -output_timestep_num = 25 +output_timestep_num = 24 auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ -erase_auxiliary_files = 0 +erase_auxiliary_files = 1 [DOMAIN] @@ -65,6 +65,8 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver # CATALUNYA test nx = 28 ny = 30 + #nx = 4 + #ny = 4 inc_x = 10000 inc_y = 10000 x_0 = 253151.59375 @@ -122,7 +124,7 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver # SECTORS #################################################################### [SECTOR MANAGEMENT] -writing_processors = 12 +writing_processors = 1 # aviation_processors = 1 # shipping_port_processors = 1 @@ -140,18 +142,18 @@ aviation_processors = 0 shipping_port_processors = 0 livestock_processors = 0 crop_operations_processors = 0 -crop_fertilizers_processors = 0 +crop_fertilizers_processors = 1 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 -point_sources_processors = 1 +point_sources_processors = 0 traffic_processors = 0 traffic_area_processors = 0 [SHAPEFILES] -nut_shapefile_prov = /Shapefiles/Provinces/ES_Provinces.shp -nut_shapefile_ccaa = /Shapefiles/CCAA/ES_CCAA.shp +nuts3_shapefile = /shapefiles/nuts3/nuts3.shp +nuts2_shapefile = /shapefiles/nuts2/nuts2.shp population_density_map = /jrc/ghsl/original_files/GHS_POP_GPW42015_GLOBE_R2015A_54009_1k_v1_0.tif [SPECIATION DATA] @@ -198,8 +200,8 @@ aviation_speciation_profiles = /profiles/speciation/aviation/speciati shipping_port_source_pollutants = nox_no2, pm10, pm25, co, so2, nmvoc, ch4, nh3, co2 vessel_list = LC,DC,GC,RO,FE,CR,CO,TU,OT port_list = ACO, ALC, ALI, ALM, ARI, ARR, AVI, ALG, BAR, BIL, CAB, CAD, CSA, CAR, CTG, CAS, CEU, HIE, FER, GAN, GIJ, HUE, IBI, LPM, LCR, MAH, MAL, MPO, MEL, MOT, PMA, PAS, PRO, PSM, SSG, SCP, SCT, SAG, SAL, SCI, SAN, SEV, TAR, TRG, VAL, VIG, VIL, ZFC -hoteling_shapefile_path = /Shapefiles/shipping_port/Areas_Hot_Puertos_and_BCN.shp -maneuvering_shapefile_path = /Shapefiles/shipping_port/Areas_Maneuv_Puertos_and_BCN.shp +hoteling_shapefile_path = /shapefiles/shipping_port/Areas_Hot_Puertos_and_BCN.shp +maneuvering_shapefile_path = /shapefiles/shipping_port/Areas_Maneuv_Puertos_and_BCN.shp shipping_port_ef_path = /shipping_port/ef/engines_fuel_EF.csv shipping_port_engine_percent_path = /shipping_port/ef/ship_perc_engines.csv shipping_port_tonnage_path = /shipping_port/ship_operations_GT_2015.csv @@ -267,7 +269,7 @@ crop_machinery_monthly_profiles = /profiles/temporal/agricultural_mac crop_machinery_weekly_profiles = /profiles/temporal/agricultural_machinery/weekly_profiles.csv crop_machinery_hourly_profiles = /profiles/temporal/agricultural_machinery/hourly_profiles.csv crop_machinery_speciation_profiles = /profiles/speciation/agricultural_machinery/speciation_profiles_base.csv -crop_machinery_by_nut = /agriculture/agricultural_machinery/crops_ha_prov_2017.csv +crop_machinery_nuts3 = /agriculture/agricultural_machinery/crops_ha_prov_2017.csv [RESIDENTIAL] fuel_list = HD_res, LPG_res, NG_res, HD_com, LPG_com, NG_com, B_res, B_com @@ -275,10 +277,10 @@ fuel_list = HD_res, LPG_res, NG_res, HD_com, LPG_com, NG_com, B_res, B_com # fuel_list = HD_res, LPG_res, NG_res, HD_com, LPG_com, NG_com residential_source_pollutants = nox_no2, so2, co, nh3, pm10, pm25, nmvoc population_type_map = /jrc/ghsl/original_files/GHS_SMOD_POP2015_GLOBE_R2016A_54009_1k_v1_0.tif -population_type_by_ccaa = /residential/pop_type_ccaa.csv -population_type_by_prov = /residential/pop_type_prov.csv -energy_consumption_by_prov = /residential/energy_consumption_nuts3.csv -energy_consumption_by_ccaa = /residential/energy_consumption_nuts2.csv +population_type_nuts2 = /residential/pop_type_ccaa.csv +population_type_nuts3 = /residential/pop_type_prov.csv +energy_consumption_nuts2 = /residential/energy_consumption_nuts2.csv +energy_consumption_nuts3 = /residential/energy_consumption_nuts3.csv residential_spatial_proxies = /residential/spatial_proxies.csv residential_ef_files_path = /residential/ef/ef.csv residential_heating_degree_day_path = /ecmwf/era5/yearly/heatingdegreeday/hdd_.nc @@ -298,7 +300,7 @@ recreational_boats_speciation_profiles = /profiles/speciation/recreat [POINT SOURCES] point_source_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25,ch4,n2o,co2 -plume_rise = False +plume_rise = True # point_source_snaps = 09 point_source_catalog = /point_sources/Maestra_focos_2015_plume_rise.shp point_source_monthly_profiles = /profiles/temporal/point_sources/monthly_profiles.csv @@ -331,22 +333,22 @@ traffic_hourly_profiles_mean = /profiles/temporal/traffic/aadt_h_mn.c traffic_hourly_profiles_weekday = /profiles/temporal/traffic/aadt_h_wd.csv traffic_hourly_profiles_saturday = /profiles/temporal/traffic/aadt_h_sat.csv traffic_hourly_profiles_sunday = /profiles/temporal/traffic/aadt_h_sun.csv -traffic_speciation_profile_hot_cold = /profiles/speciation/traffic/hot_cold_cmaq_cb05_aero5.csv -traffic_speciation_profile_tyre = /profiles/speciation/traffic/tyre_cmaq_cb05_aero5.csv -traffic_speciation_profile_road = /profiles/speciation/traffic/road_cmaq_cb05_aero5.csv -traffic_speciation_profile_brake = /profiles/speciation/traffic/brake_cmaq_cb05_aero5.csv -traffic_speciation_profile_resuspension = /profiles/speciation/traffic/resuspension_cmaq_cb05_aero5.csv +traffic_speciation_profile_hot_cold = /profiles/speciation/traffic/hot_cold_base.csv +traffic_speciation_profile_tyre = /profiles/speciation/traffic/tyre_base.csv +traffic_speciation_profile_road = /profiles/speciation/traffic/road_base.csv +traffic_speciation_profile_brake = /profiles/speciation/traffic/brake_base.csv +traffic_speciation_profile_resuspension = /profiles/speciation/traffic/resuspension_base.csv [TRAFFIC AREA SECTOR] traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 do_evaporative = 1 traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csv -population_by_municipality = /traffic_area/population_by_mun.csv -traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_cmaq.csv +population_nuts3 = /traffic_area/population_by_mun.csv +traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_base.csv traffic_area_evaporative_ef_file = /traffic_area/ef/evaporative_nmvoc.csv do_small_cities = 1 -traffic_area_small_cities_path = /Shapefiles/small_cities/small_cities.shp -traffic_area_speciation_profiles_small_cities = /profiles/speciation/traffic_area/small_cities_cmaq.csv +traffic_area_small_cities_path = /shapefiles/small_cities/small_cities.shp +traffic_area_speciation_profiles_small_cities = /profiles/speciation/traffic_area/small_cities_base.csv traffic_area_small_cities_ef_file = /traffic_area/ef/small_cities.csv small_cities_monthly_profile = /profiles/temporal/traffic_area/small_cities_monthly_profiles.csv small_cities_weekly_profile = /profiles/temporal/traffic_area/small_cities_weekly_profiles.csv diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 5ed5bd7ce31dccdb04bf83bd58eac4544159cf34..3c9b6bd08cba30e5d10e88da9286354438516fd7 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -167,8 +167,8 @@ class Config(ArgParser): p.add_argument('--speciation_map', required=False, help='...') # ===== SHAPEFILES ===== - p.add_argument('--nut_shapefile_prov', required=False, type=str, default='True') - p.add_argument('--nut_shapefile_ccaa', required=False, type=str, default='True') + p.add_argument('--nuts3_shapefile', required=False, type=str, default='True') + p.add_argument('--nuts2_shapefile', required=False, type=str, default='True') p.add_argument('--clipping', required=False, type=str, default=None, help='To clip the domain into an specific zone. ' + @@ -281,17 +281,17 @@ class Config(ArgParser): p.add_argument('--crop_machinery_hourly_profiles', required=False, help='...') p.add_argument('--crop_machinery_speciation_map', required=False, help='...') p.add_argument('--crop_machinery_speciation_profiles', required=False, help='...') - p.add_argument('--crop_machinery_by_nut', required=False, help='...') + p.add_argument('--crop_machinery_nuts3', required=False, help='...') # ***** RESIDENTIAL SECTOR ***** p.add_argument('--fuel_list', required=False, help='...') p.add_argument('--residential_source_pollutants', required=False, help='...') p.add_argument('--population_density_map', required=False, help='...') p.add_argument('--population_type_map', required=False, help='...') - p.add_argument('--population_type_by_ccaa', required=False, help='...') - p.add_argument('--population_type_by_prov', required=False, help='...') - p.add_argument('--energy_consumption_by_prov', required=False, help='...') - p.add_argument('--energy_consumption_by_ccaa', required=False, help='...') + p.add_argument('--population_type_nuts2', required=False, help='...') + p.add_argument('--population_type_nuts3', required=False, help='...') + p.add_argument('--energy_consumption_nuts3', required=False, help='...') + p.add_argument('--energy_consumption_nuts2', required=False, help='...') p.add_argument('--residential_spatial_proxies', required=False, help='...') p.add_argument('--residential_ef_files_path', required=False, help='...') p.add_argument('--residential_heating_degree_day_path', required=False, help='...') @@ -353,7 +353,7 @@ class Config(ArgParser): p.add_argument('--traffic_area_pollutants', required=False, help='...') p.add_argument('--do_evaporative', required=False, help='...') p.add_argument('--traffic_area_gas_path', required=False, help='...') - p.add_argument('--population_by_municipality', required=False, help='...') + p.add_argument('--population_nuts3', required=False, help='...') p.add_argument('--traffic_area_speciation_profiles_evaporative', required=False, help='...') p.add_argument('--traffic_area_evaporative_ef_file', required=False, help='...') p.add_argument('--do_small_cities', required=False, help='...') diff --git a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py index 25aeec0db276c7f907f17d8dea1755b3d9612f32..924d70dad9f344f526e1d63fa3afaef50609e012 100755 --- a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py @@ -32,7 +32,9 @@ class AgriculturalCropFertilizersSector(AgriculturalSector): self.cultivated_ratio = self.read_profiles(cultivated_ratio) self.fertilizer_rate = self.read_profiles(fertilizer_rate) + self.fertilizer_rate.rename(columns={'nuts2_id': 'code'}, inplace=True) self.crop_f_parameter = self.read_profiles(crop_f_parameter) + self.crop_f_parameter.rename(columns={'nuts2_id': 'code'}, inplace=True) self.crop_f_fertilizers = self.read_profiles(crop_f_fertilizers) if self.comm.Get_rank() == 0: diff --git a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py index bb2386098ae80c1d4cf14844a98d1bedfd55ff9d..9930f47b0ad2ac59b61b92719d9c3d58d4782a1a 100755 --- a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py @@ -31,7 +31,7 @@ class AgriculturalCropOperationsSector(AgriculturalSector): :type date_array: list(datetime.datetime, ...) :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER06' information with the NUT_code. + the 'nuts2_id' information with the NUT_code. :type nut_shapefile_path: str :param source_pollutants: List of input pollutants to take into account. Agricultural livestock module can @@ -84,7 +84,7 @@ class AgriculturalCropOperationsSector(AgriculturalSector): :param crop_by_nut: :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT_code. + the 'nuts3_id' information with the NUT_code. :type nut_shapefile_path: str """ spent_time = timeit.default_timer() diff --git a/hermesv3_bu/sectors/agricultural_machinery_sector.py b/hermesv3_bu/sectors/agricultural_machinery_sector.py index 3dcd17bfe5e1d0798f85b7eac6b478d31ac5668e..12a7be3db1cc7fc55a5cf0930f3f64ce40ecf34f 100755 --- a/hermesv3_bu/sectors/agricultural_machinery_sector.py +++ b/hermesv3_bu/sectors/agricultural_machinery_sector.py @@ -21,7 +21,7 @@ class AgriculturalMachinerySector(AgriculturalSector): speciation_profiles_path, molecular_weights_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, machinery_distibution_nut_shapefile_path, deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, vehicle_workhours_path, vehicle_power_path, - crop_machinery_by_nut): + crop_machinery_nuts3): spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL MACHINERY SECTOR =====') @@ -32,10 +32,10 @@ class AgriculturalMachinerySector(AgriculturalSector): speciation_profiles_path, molecular_weights_path) self.machinery_list = machinery_list - self.crop_machinery_by_nut = self.read_profiles(crop_machinery_by_nut) + self.crop_machinery_nuts3 = self.read_profiles(crop_machinery_nuts3) self.crop_distribution = self.get_crop_distribution_by_nut( - self.crop_distribution, machinery_distibution_nut_shapefile_path, nut_code='ORDER07') + self.crop_distribution, machinery_distibution_nut_shapefile_path, nut_code='nuts3_id') self.months = self.get_date_array_by_month() @@ -53,8 +53,8 @@ class AgriculturalMachinerySector(AgriculturalSector): spent_time = timeit.default_timer() def get_fraction(dataframe): - total_crop_sum = self.crop_machinery_by_nut.loc[self.crop_machinery_by_nut[nut_code] == int(dataframe.name), - self.crop_list].values.sum() + total_crop_sum = self.crop_machinery_nuts3.loc[self.crop_machinery_nuts3[nut_code] == int(dataframe.name), + self.crop_list].values.sum() dataframe['fraction'] = dataframe[self.crop_list].sum(axis=1) / total_crop_sum return dataframe.loc[:, ['fraction']] @@ -81,7 +81,7 @@ class AgriculturalMachinerySector(AgriculturalSector): if write_crop_by_nut: crop_distribution.loc[:, self.crop_list + [nut_code]].groupby(nut_code).sum().reset_index().to_csv( - self.crop_machinery_by_nut) + self.crop_machinery_nuts3) crop_distribution['fraction'] = crop_distribution.groupby(nut_code).apply(get_fraction) crop_distribution.drop(columns=self.crop_list, inplace=True) crop_distribution.rename(columns={nut_code: 'NUT_code'}, inplace=True) @@ -112,30 +112,31 @@ class AgriculturalMachinerySector(AgriculturalSector): spent_time = timeit.default_timer() def get_n(df): - df['N'] = self.vehicle_units.loc[self.vehicle_units['code'] == df.name[0], df.name[1]].values[0] + df['N'] = self.vehicle_units.loc[self.vehicle_units['nuts3_id'] == df.name[0], df.name[1]].values[0] return df.loc[:, ['N']] def get_s(df): df['S'] = self.vehicle_ratio.loc[ - (self.vehicle_ratio['code'] == df.name[0]) & (self.vehicle_ratio['technology'] == df.name[2]), + (self.vehicle_ratio['nuts3_id'] == df.name[0]) & (self.vehicle_ratio['technology'] == df.name[2]), df.name[1]].values[0] return df.loc[:, ['S']] def get_t(df): try: - df['T'] = self.vehicle_workhours.loc[(self.vehicle_workhours['code'] == df.name[0]) & + df['T'] = self.vehicle_workhours.loc[(self.vehicle_workhours['nuts3_id'] == df.name[0]) & (self.vehicle_workhours['technology'] == df.name[2]), df.name[1]].values[0] except IndexError: df['T'] = np.nan df.loc[df['T'].isna(), 'T'] = self.vehicle_workhours.loc[ - (self.vehicle_workhours['code'] == df.name[0]) & (self.vehicle_workhours['technology'] == 'default'), + (self.vehicle_workhours['nuts3_id'] == df.name[0]) & (self.vehicle_workhours['technology'] == + 'default'), df.name[1]].values[0] return df.loc[:, ['T']] def get_p(df): - df['P'] = self.vehicle_power.loc[self.vehicle_power['code'] == df.name[0], df.name[1]].values[0] + df['P'] = self.vehicle_power.loc[self.vehicle_power['nuts3_id'] == df.name[0], df.name[1]].values[0] return df.loc[:, ['P']] def get_lf(df): diff --git a/hermesv3_bu/sectors/agricultural_sector.py b/hermesv3_bu/sectors/agricultural_sector.py index be849801a979e470eb9d4eca5a727f9bda115e42..00f1973999029abcac6895cfafef2f6a2ddc6d1f 100755 --- a/hermesv3_bu/sectors/agricultural_sector.py +++ b/hermesv3_bu/sectors/agricultural_sector.py @@ -230,8 +230,8 @@ class AgriculturalSector(Sector): land_uses_shp = IoRaster(self.comm_agr).to_shapefile_serie(land_uses_clipped) ccaa_shp = IoShapefile(self.comm_agr).read_shapefile_serial(self.nut_shapefile).to_crs(land_uses_shp.crs) - ccaa_shp.drop(columns=['NAME', 'ORDER06'], inplace=True) - ccaa_shp.rename(columns={'CODE': 'NUT'}, inplace=True) + ccaa_shp.drop(columns=['nuts2_na'], inplace=True) + ccaa_shp.rename(columns={'nuts2_id': 'NUT'}, inplace=True) # ccaa_shp.set_index('NUT', inplace=True) land_use_src_by_nut = self.spatial_overlays(land_uses_shp.reset_index(), ccaa_shp, how='intersection') land_use_src_by_nut.drop(columns=['idx1', 'idx2', 'CELL_ID'], inplace=True) @@ -258,7 +258,8 @@ class AgriculturalSector(Sector): """ spent_time = timeit.default_timer() - df = pd.read_csv(self.land_use_by_nut) + df = pd.read_csv(self.land_use_by_nut, dtype={'nuts2_id': str}) + df.rename(columns={'nuts2_id': 'NUT'}, inplace=True) df = df.loc[df['land_use'].isin(land_uses), :] df.set_index(['NUT', 'land_use'], inplace=True) @@ -280,11 +281,12 @@ class AgriculturalSector(Sector): """ spent_time = timeit.default_timer() - land_use_distribution_src_nut['area'] = land_use_distribution_src_nut.area - land_use_by_nut = land_use_distribution_src_nut.groupby(['NUT', 'land_use']).sum().reset_index() - land_use_by_nut = land_use_by_nut.loc[land_use_by_nut['land_use'].isin(land_uses), :] + land_use_by_nut = pd.DataFrame(index=pd.MultiIndex.from_product( + [np.unique(land_use_distribution_src_nut['NUT']), land_uses], names=['NUT', 'land_use'])) - land_use_by_nut.set_index(['NUT', 'land_use'], inplace=True) + land_use_distribution_src_nut['area'] = land_use_distribution_src_nut.area + land_use_by_nut['area'] = land_use_distribution_src_nut.groupby(['NUT', 'land_use']).sum() + land_use_by_nut.fillna(0.0, inplace=True) self.logger.write_time_log('AgriculturalSector', 'get_land_use_by_nut_csv', timeit.default_timer() - spent_time) return land_use_by_nut @@ -303,9 +305,9 @@ class AgriculturalSector(Sector): :rtype: DataFrame """ spent_time = timeit.default_timer() - if nuts is not None: - land_use_by_nut = land_use_by_nut.loc[nuts, :] + land_use_by_nut = land_use_by_nut.iloc[land_use_by_nut.index.get_level_values('NUT').isin(nuts)] + new_df = pd.DataFrame(index=np.unique(land_use_by_nut.index.get_level_values('NUT')), columns=self.crop_from_landuse.keys()) new_df.fillna(0, inplace=True) @@ -336,6 +338,7 @@ class AgriculturalSector(Sector): crop_share_by_nut = crop_by_nut.copy() for crop in crop_by_nut.columns: crop_share_by_nut[crop] = crop_by_nut[crop] / tot_crop_by_nut[crop] + self.logger.write_time_log('AgriculturalSector', 'get_crop_shape_by_nut', timeit.default_timer() - spent_time) return crop_share_by_nut @@ -353,11 +356,13 @@ class AgriculturalSector(Sector): """ spent_time = timeit.default_timer() - crop_by_nut = pd.read_csv(self.crop_by_nut) - crop_by_nut.drop(columns='name', inplace=True) + crop_by_nut = pd.read_csv(self.crop_by_nut, dtype={'nuts2_id': str}) + crop_by_nut.drop(columns='nuts2_na', inplace=True) + crop_by_nut.rename(columns={'nuts2_id': 'code'}, inplace=True) - crop_by_nut['code'] = crop_by_nut['code'].astype(np.int16) + # crop_by_nut['code'] = crop_by_nut['code'].astype(np.int16) crop_by_nut.set_index('code', inplace=True) + crop_by_nut = crop_by_nut.loc[crop_share_by_nut.index, :] crop_area_by_nut = crop_share_by_nut * crop_by_nut @@ -455,7 +460,6 @@ class AgriculturalSector(Sector): message_level=2) involved_land_uses = self.get_involved_land_uses() - land_use_distribution_src_nut = self.get_land_use_src_by_nut(involved_land_uses) land_use_by_nut = self.get_land_use_by_nut_csv(land_use_distribution_src_nut, involved_land_uses) tot_land_use_by_nut = self.get_tot_land_use_by_nut(involved_land_uses) @@ -463,6 +467,7 @@ class AgriculturalSector(Sector): crop_by_nut = self.land_use_to_crop_by_nut(land_use_by_nut) tot_crop_by_nut = self.land_use_to_crop_by_nut( tot_land_use_by_nut, nuts=list(np.unique(land_use_by_nut.index.get_level_values('NUT')))) + crop_shape_by_nut = self.get_crop_shape_by_nut(crop_by_nut, tot_crop_by_nut) crop_area_by_nut = self.get_crop_area_by_nut(crop_shape_by_nut) diff --git a/hermesv3_bu/sectors/livestock_sector.py b/hermesv3_bu/sectors/livestock_sector.py index 80d4b35a50f0d51e1cd843a59ef5a369a00d3895..8bf264d534125be1ddb94347c037736219798adc 100755 --- a/hermesv3_bu/sectors/livestock_sector.py +++ b/hermesv3_bu/sectors/livestock_sector.py @@ -148,7 +148,7 @@ class LivestockSector(Sector): :type molecular_weights_path: str :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT_code. + the 'nuts3_id' information with the NUT_code. :type nut_shapefile_path: str """ spent_time = timeit.default_timer() @@ -187,14 +187,14 @@ class LivestockSector(Sector): :type gridded_livestock_path: str :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT ID. + the 'nuts3_id' information with the NUT ID. :type nut_shapefile_path: str :param correction_split_factors_path: Path to the CSV file that contains the correction factors and the splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["NUT", "nut_code", "_fact", "_01", ...] + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01",...] "nut_code" column must contain the NUT ID. :type correction_split_factors_path: str @@ -383,7 +383,7 @@ class LivestockSector(Sector): splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["NUT", "nut_code", "_fact", "_01", ...] + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01",...] "nut_code" column must contain the NUT ID. :type correction_split_factors_path: str @@ -394,20 +394,20 @@ class LivestockSector(Sector): splitting_factors_list = [] for animal in self.animal_list: correction_split_factors = pd.read_csv(correction_split_factors_path.replace('', animal)) - correction_split_factors.set_index('nut_code', inplace=True) + correction_split_factors.set_index('nuts3_id', inplace=True) categories = list(correction_split_factors.columns.values) - categories = [e for e in categories if e not in ['NUT', 'nut_code', '{0}_fact'.format(animal)]] + categories = [e for e in categories if e not in ['nuts3_na', 'nuts3_id', '{0}_fact'.format(animal)]] correction_split_factors[categories] = correction_split_factors.loc[:, categories].multiply( correction_split_factors['{0}_fact'.format(animal)], axis='index') - correction_split_factors.drop(columns=['NUT', '{0}_fact'.format(animal)], inplace=True) + correction_split_factors.drop(columns=['nuts3_na', '{0}_fact'.format(animal)], inplace=True) splitting_factors_list.append(correction_split_factors) splitting_factors = pd.concat(splitting_factors_list, axis=1) splitting_factors.reset_index(inplace=True) - splitting_factors['nut_code'] = splitting_factors['nut_code'].astype(np.int16) + splitting_factors['nuts3_id'] = splitting_factors['nuts3_id'].astype(np.int16) self.logger.write_time_log('LivestockSector', 'get_splitting_factors', timeit.default_timer() - spent_time) return splitting_factors @@ -420,15 +420,15 @@ class LivestockSector(Sector): :type dataframe: geopandas.GeoDataframe :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT_code. + the 'nuts3_id' information with the NUT_code. :type nut_shapefile_path: str :param correction_split_factors_path: Path to the CSV file that contains the correction factors and the splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["NUT", "nut_code", "_fact", "_01", + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01", ...] - "nut_code" column must contain the NUT ID + "nuts3_id" column must contain the NUT ID :type correction_split_factors_path: str :return: GeoDataframe with the amount of each animal subtype by destiny cell (FID) @@ -444,14 +444,15 @@ class LivestockSector(Sector): 'animal_distribution_by_cat.shp') if not os.path.exists(animal_distribution_path): - dataframe = self.add_nut_code(dataframe, nut_shapefile_path, nut_value='ORDER07') + dataframe = self.add_nut_code(dataframe, nut_shapefile_path, nut_value='nuts3_id') + dataframe.rename(columns={'nut_code': 'nuts3_id'}, inplace=True) splitting_factors = self.get_splitting_factors(correction_split_factors_path) # Adding the splitting factors by NUT code - dataframe = pd.merge(dataframe, splitting_factors, how='left', on='nut_code') + dataframe = pd.merge(dataframe, splitting_factors, how='left', on='nuts3_id') - dataframe.drop(columns=['nut_code'], inplace=True) + dataframe.drop(columns=['nuts3_id'], inplace=True) for animal in self.animal_list: animal_types = [i for i in list(dataframe.columns.values) if i.startswith(animal)] diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index 03463400c71f5919c0eaf83925846dc21bfb20fd..a13d765be88a2b90e5dadbddbb62276b2ef81578 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -440,7 +440,7 @@ class PointSourceSector(Sector): var = netcdf.variables[var_name][time_index, 0, :] netcdf.close() - dataframe[var_name] = var[dataframe['X'], dataframe['Y']] + dataframe[var_name] = var[dataframe['Y'], dataframe['X']] return dataframe[[var_name]] @@ -457,7 +457,7 @@ class PointSourceSector(Sector): var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] pre_t_lay = 0 lay_list = [] @@ -488,7 +488,7 @@ class PointSourceSector(Sector): var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] lay_list = ['temp_sfc'] for i, t_lay in enumerate(var): @@ -521,7 +521,7 @@ class PointSourceSector(Sector): var = u10_netcdf.variables[u_var_name][time_index, 0, :] u10_netcdf.close() - dataframe['u10'] = var[dataframe['X'], dataframe['Y']] + dataframe['u10'] = var[dataframe['Y'], dataframe['X']] # === v10 === v10_nc_path = os.path.join( @@ -530,7 +530,7 @@ class PointSourceSector(Sector): var = v10_netcdf.variables[v_var_name][time_index, 0, :] v10_netcdf.close() - dataframe['v10'] = var[dataframe['X'], dataframe['Y']] + dataframe['v10'] = var[dataframe['Y'], dataframe['X']] # === wind speed === dataframe['wSpeed_10'] = np.linalg.norm(dataframe[['u10', 'v10']].values, axis=1) @@ -552,7 +552,7 @@ class PointSourceSector(Sector): var = np.flipud(u10_netcdf.variables[u_var_name][time_index, :, :, :]) u10_netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] for i, t_lay in enumerate(var): dataframe['u_{0}'.format(i)] = t_lay @@ -564,7 +564,7 @@ class PointSourceSector(Sector): var = np.flipud(v10_netcdf.variables[v_var_name][time_index, :, :, :]) v10_netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] ws_lay_list = ['wSpeed_10'] for i, t_lay in enumerate(var): diff --git a/hermesv3_bu/sectors/residential_sector.py b/hermesv3_bu/sectors/residential_sector.py index 22cd1161fc38bfa2c62f097f7acb35c22a016252..495b5e7d10024e98f0f9b0728ad74201904efd32 100755 --- a/hermesv3_bu/sectors/residential_sector.py +++ b/hermesv3_bu/sectors/residential_sector.py @@ -18,8 +18,8 @@ from hermesv3_bu.logger.log import Log class ResidentialSector(Sector): def __init__(self, comm, logger, auxiliary_dir, grid_shp, clip, date_array, source_pollutants, vertical_levels, fuel_list, prov_shapefile, ccaa_shapefile, population_density_map, population_type_map, - population_type_by_ccaa, population_type_by_prov, energy_consumption_by_prov, - energy_consumption_by_ccaa, residential_spatial_proxies, residential_ef_files_path, + population_type_nuts2, population_type_nuts3, energy_consumption_nuts3, + energy_consumption_nuts2, residential_spatial_proxies, residential_ef_files_path, heating_degree_day_path, temperature_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path): spent_time = timeit.default_timer() @@ -37,11 +37,11 @@ class ResidentialSector(Sector): self.fuel_list = fuel_list self.day_dict = self.calculate_num_days() - self.pop_type_by_prov = population_type_by_prov - self.pop_type_by_ccaa = population_type_by_ccaa + self.pop_type_by_prov = population_type_nuts3 + self.pop_type_by_ccaa = population_type_nuts2 - self.energy_consumption_by_prov = pd.read_csv(energy_consumption_by_prov) - self.energy_consumption_by_ccaa = pd.read_csv(energy_consumption_by_ccaa) + self.energy_consumption_nuts3 = pd.read_csv(energy_consumption_nuts3) + self.energy_consumption_nuts2 = pd.read_csv(energy_consumption_nuts2) self.residential_spatial_proxies = self.read_residential_spatial_proxies(residential_spatial_proxies) self.ef_profiles = self.read_ef_file(residential_ef_files_path) @@ -173,10 +173,10 @@ class ResidentialSector(Sector): # population_density = gpd.sjoin(population_density, population_type, how='left', op='intersects') # population_density.drop(columns=['index_right'], inplace=True) - population_density = self.add_nut_code(population_density, prov_shapefile, nut_value='ORDER07') + population_density = self.add_nut_code(population_density, prov_shapefile, nut_value='nuts3_id') population_density.rename(columns={'nut_code': 'prov'}, inplace=True) population_density = population_density.loc[population_density['prov'] != -999, :] - population_density = self.add_nut_code(population_density, ccaa_shapefile, nut_value='ORDER06') + population_density = self.add_nut_code(population_density, ccaa_shapefile, nut_value='nuts2_id') population_density.rename(columns={'nut_code': 'ccaa'}, inplace=True) population_density = population_density.loc[population_density['ccaa'] != -999, :] @@ -186,8 +186,10 @@ class ResidentialSector(Sector): population_density.loc[:, ['ccaa', 'pop', 'type']].groupby(['ccaa', 'type']).sum().reset_index().to_csv( self.pop_type_by_ccaa) - self.pop_type_by_ccaa = pd.read_csv(self.pop_type_by_ccaa).set_index(['ccaa', 'type']) - self.pop_type_by_prov = pd.read_csv(self.pop_type_by_prov).set_index(['prov', 'type']) + self.pop_type_by_ccaa = pd.read_csv(self.pop_type_by_ccaa).rename( + columns={'nuts2_id': 'ccaa'}).set_index(['ccaa', 'type']) + self.pop_type_by_prov = pd.read_csv(self.pop_type_by_prov).rename( + columns={'nuts3_id': 'prov'}).set_index(['prov', 'type']) fuel_distribution = population_density[['geometry']].copy() fuel_distribution.index.name = 'CELL_ID' @@ -202,8 +204,8 @@ class ResidentialSector(Sector): if spatial_proxy['proxy_type'] == 'all': total_pop = self.pop_type_by_ccaa.loc[ self.pop_type_by_ccaa.index.get_level_values('ccaa') == ccaa, 'pop'].sum() - energy_consumption = self.energy_consumption_by_ccaa.loc[ - self.energy_consumption_by_ccaa['code'] == ccaa, fuel].values[0] + energy_consumption = self.energy_consumption_nuts2.loc[ + self.energy_consumption_nuts2['nuts2_id'] == ccaa, fuel].values[0] fuel_distribution.loc[ population_density['ccaa'] == ccaa, fuel] = population_density['pop'].multiply( @@ -213,8 +215,8 @@ class ResidentialSector(Sector): (self.pop_type_by_ccaa.index.get_level_values('ccaa') == ccaa) & (self.pop_type_by_ccaa.index.get_level_values('type') == spatial_proxy['proxy_type']), 'pop'].values[0] - energy_consumption = self.energy_consumption_by_ccaa.loc[ - self.energy_consumption_by_ccaa['code'] == ccaa, fuel].values[0] + energy_consumption = self.energy_consumption_nuts2.loc[ + self.energy_consumption_nuts2['nuts2_id'] == ccaa, fuel].values[0] fuel_distribution.loc[(population_density['ccaa'] == ccaa) & (population_density['type'] == spatial_proxy['proxy_type']), @@ -225,8 +227,8 @@ class ResidentialSector(Sector): if spatial_proxy['proxy_type'] == 'all': total_pop = self.pop_type_by_prov.loc[self.pop_type_by_prov.index.get_level_values( 'prov') == prov, 'pop'].sum() - energy_consumption = self.energy_consumption_by_prov.loc[ - self.energy_consumption_by_prov['code'] == prov, fuel].values[0] + energy_consumption = self.energy_consumption_nuts3.loc[ + self.energy_consumption_nuts3['nuts3_id'] == prov, fuel].values[0] fuel_distribution.loc[population_density['prov'] == prov, fuel] = population_density[ 'pop'].multiply(energy_consumption / total_pop) @@ -235,21 +237,19 @@ class ResidentialSector(Sector): (self.pop_type_by_prov.index.get_level_values('prov') == prov) & (self.pop_type_by_prov.index.get_level_values('type') == spatial_proxy['proxy_type']), 'pop'].values[0] - energy_consumption = self.energy_consumption_by_prov.loc[ - self.energy_consumption_by_prov['code'] == prov, fuel].values[0] + energy_consumption = self.energy_consumption_nuts3.loc[ + self.energy_consumption_nuts3['nuts3_id'] == prov, fuel].values[0] fuel_distribution.loc[(population_density['prov'] == prov) & (population_density['type'] == spatial_proxy['proxy_type']), fuel] = population_density['pop'].multiply( energy_consumption / total_pop) fuel_distribution = self.to_dst_resolution(fuel_distribution) - + fuel_distribution.set_index('FID', inplace=True) IoShapefile(self.comm).write_shapefile_serial(fuel_distribution.reset_index(), fuel_distribution_path) else: fuel_distribution = IoShapefile(self.comm).read_shapefile_serial(fuel_distribution_path) - fuel_distribution.drop(columns=['index'], inplace=True) - - fuel_distribution.set_index('FID', inplace=True) + fuel_distribution.set_index('FID', inplace=True) self.logger.write_time_log('ResidentialSector', 'get_fuel_distribution', timeit.default_timer() - spent_time) return fuel_distribution diff --git a/hermesv3_bu/sectors/sector.py b/hermesv3_bu/sectors/sector.py index edcf9555bcf50416df96d792e2edadc6e855469f..8aae0b543ceaf68275c9e21ec6f95b63ba603364 100755 --- a/hermesv3_bu/sectors/sector.py +++ b/hermesv3_bu/sectors/sector.py @@ -421,7 +421,7 @@ class Sector(object): return dataframe - def add_nut_code(self, shapefile, nut_shapefile_path, nut_value='ORDER06'): + def add_nut_code(self, shapefile, nut_shapefile_path, nut_value='nuts2_id'): """ Add 'nut_code' column into the shapefile based on the 'nut_value' column of the 'nut_shapefile_path' shapefile. diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index b306dca36ff8caea46bcd11aa26100554a367aef..048eeb6e35485f3112674a389506d995f4d452bf 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -70,7 +70,7 @@ class SectorManager(object): arguments.denominator_yearly_factor_dir, arguments.livestock_ef_files_dir, arguments.livestock_monthly_profiles, arguments.livestock_weekly_profiles, arguments.livestock_hourly_profiles, arguments.speciation_map, - arguments.livestock_speciation_profiles, arguments.molecular_weights, arguments.nut_shapefile_prov) + arguments.livestock_speciation_profiles, arguments.molecular_weights, arguments.nuts3_shapefile) elif sector == 'crop_operations' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.agricultural_crop_operations_sector import AgriculturalCropOperationsSector @@ -80,7 +80,7 @@ class SectorManager(object): self.sector = AgriculturalCropOperationsSector( comm_agr, comm, logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.crop_operations_source_pollutants, - grid.vertical_desctiption, arguments.crop_operations_list, arguments.nut_shapefile_ccaa, + grid.vertical_desctiption, arguments.crop_operations_list, arguments.nuts2_shapefile, arguments.land_uses_path, arguments.crop_operations_ef_files_dir, arguments.crop_operations_monthly_profiles, arguments.crop_operations_weekly_profiles, arguments.crop_operations_hourly_profiles, arguments.speciation_map, @@ -95,7 +95,7 @@ class SectorManager(object): self.sector = AgriculturalCropFertilizersSector( comm_agr, comm, logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.crop_fertilizers_source_pollutants, grid.vertical_desctiption, - arguments.crop_fertilizers_list, arguments.nut_shapefile_ccaa, arguments.land_uses_path, + arguments.crop_fertilizers_list, arguments.nuts2_shapefile, arguments.land_uses_path, arguments.crop_fertilizers_hourly_profiles, arguments.speciation_map, arguments.crop_fertilizers_speciation_profiles, arguments.molecular_weights, arguments.land_use_by_nut_path, arguments.crop_by_nut_path, arguments.crop_from_landuse_path, @@ -113,16 +113,16 @@ class SectorManager(object): self.sector = AgriculturalMachinerySector( comm_agr, comm, logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.crop_machinery_source_pollutants, grid.vertical_desctiption, - arguments.crop_machinery_list, arguments.nut_shapefile_ccaa, arguments.machinery_list, + arguments.crop_machinery_list, arguments.nuts2_shapefile, arguments.machinery_list, arguments.land_uses_path, arguments.crop_machinery_ef_path, arguments.crop_machinery_monthly_profiles, arguments.crop_machinery_weekly_profiles, arguments.crop_machinery_hourly_profiles, arguments.speciation_map, arguments.crop_machinery_speciation_profiles, arguments.molecular_weights, arguments.land_use_by_nut_path, arguments.crop_by_nut_path, arguments.crop_from_landuse_path, - arguments.nut_shapefile_prov, arguments.crop_machinery_deterioration_factor_path, + arguments.nuts3_shapefile, arguments.crop_machinery_deterioration_factor_path, arguments.crop_machinery_load_factor_path, arguments.crop_machinery_vehicle_ratio_path, arguments.crop_machinery_vehicle_units_path, arguments.crop_machinery_vehicle_workhours_path, - arguments.crop_machinery_vehicle_power_path, arguments.crop_machinery_by_nut) + arguments.crop_machinery_vehicle_power_path, arguments.crop_machinery_nuts3) elif sector == 'residential' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.residential_sector import ResidentialSector @@ -130,9 +130,9 @@ class SectorManager(object): comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.residential_source_pollutants, grid.vertical_desctiption, arguments.fuel_list, - arguments.nut_shapefile_prov, arguments.nut_shapefile_ccaa, arguments.population_density_map, - arguments.population_type_map, arguments.population_type_by_ccaa, arguments.population_type_by_prov, - arguments.energy_consumption_by_prov, arguments.energy_consumption_by_ccaa, + arguments.nuts3_shapefile, arguments.nuts2_shapefile, arguments.population_density_map, + arguments.population_type_map, arguments.population_type_nuts2, arguments.population_type_nuts3, + arguments.energy_consumption_nuts3, arguments.energy_consumption_nuts2, arguments.residential_spatial_proxies, arguments.residential_ef_files_path, arguments.residential_heating_degree_day_path, arguments.temperature_daily_files_path, arguments.residential_hourly_profiles, arguments.speciation_map, @@ -196,7 +196,7 @@ class SectorManager(object): arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.traffic_area_pollutants, grid.vertical_desctiption, arguments.population_density_map, arguments.speciation_map, arguments.molecular_weights, arguments.do_evaporative, arguments.traffic_area_gas_path, - arguments.population_by_municipality, arguments.nut_shapefile_prov, + arguments.population_nuts3, arguments.nuts3_shapefile, arguments.traffic_area_speciation_profiles_evaporative, arguments.traffic_area_evaporative_ef_file, arguments.temperature_hourly_files_path, arguments.do_small_cities, arguments.traffic_area_small_cities_path, arguments.traffic_area_speciation_profiles_small_cities, diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 75932b452578cb3ba9ed51fe811900e560b04b2c..f870161c9bb533bfd0960f4ca9cea9d37e24465f 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -115,7 +115,7 @@ class TrafficAreaSector(Sector): return population_density def make_population_by_nuts(self, population_shape, nut_shp, pop_by_nut_path, write_file=True, csv_path=None, - column_id='ORDER07'): + column_id='nuts3_id'): spent_time = timeit.default_timer() if not os.path.exists(pop_by_nut_path): @@ -142,8 +142,7 @@ class TrafficAreaSector(Sector): if not os.path.exists(pop_nut_cell_path): pop_by_nut = pop_by_nut.to_crs(self.grid_shp.crs) - - del pop_by_nut['NAME'] + # del pop_by_nut[''] pop_by_nut['area_in'] = pop_by_nut.geometry.area # df = gpd.overlay(pop_by_nut, grid_shp, how='intersection') @@ -162,7 +161,7 @@ class TrafficAreaSector(Sector): return df def make_vehicles_by_cell(self, pop_nut_cell, gasoline_path, total_pop_by_nut, veh_by_cell_path, - column_id='ORDER07'): + column_id='nuts3_id'): spent_time = timeit.default_timer() if not os.path.exists(veh_by_cell_path): @@ -172,8 +171,8 @@ class TrafficAreaSector(Sector): df = pd.merge(pop_nut_cell, total_pop_by_nut, left_on=column_id, right_on=column_id, how='left') - df['pop_percent'] = df['data_x'] / df['data_y'] - df.drop(columns=['data_x', 'data_y'], inplace=True) + df['pop_percent'] = df['data'] / df['population'] + df.drop(columns=['data', 'population'], inplace=True) gas_df = pd.read_csv(gasoline_path, index_col='COPERT_V_name').transpose() vehicle_type_list = list(gas_df.columns.values)