From b4977daaf1165134d19dad4065e4e09768008a97 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 22 Aug 2019 18:55:20 +0200 Subject: [PATCH 1/6] CHanging ccaa by nuts2 and prov by nuts3 --- conf/hermes.conf | 40 ++++++++--------- hermesv3_bu/config/config.py | 14 +++--- .../agricultural_crop_operations_sector.py | 4 +- .../sectors/agricultural_machinery_sector.py | 2 +- hermesv3_bu/sectors/agricultural_sector.py | 4 +- hermesv3_bu/sectors/livestock_sector.py | 8 ++-- hermesv3_bu/sectors/residential_sector.py | 44 +++++++++---------- hermesv3_bu/sectors/sector.py | 2 +- hermesv3_bu/sectors/sector_manager.py | 18 ++++---- hermesv3_bu/sectors/traffic_area_sector.py | 11 +++-- 10 files changed, 73 insertions(+), 74 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 5ebfb16..7a0943c 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__point_new.nc +output_name = HERMESv3__res.nc emission_summary = 1 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -142,16 +142,16 @@ livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 -residential_processors = 0 +residential_processors = 1 recreational_boats_processors = 0 -point_sources_processors = 1 +point_sources_processors = 0 traffic_processors = 0 traffic_area_processors = 0 [SHAPEFILES] -nut_shapefile_prov = /Shapefiles/Provinces/ES_Provinces.shp -nut_shapefile_ccaa = /Shapefiles/CCAA/ES_CCAA.shp +nuts3_shapefile = /shapefiles/nuts3/nuts3.shp +nuts2_shapefile = /shapefiles/nuts2/nuts2.shp population_density_map = /jrc/ghsl/original_files/GHS_POP_GPW42015_GLOBE_R2015A_54009_1k_v1_0.tif [SPECIATION DATA] @@ -198,8 +198,8 @@ aviation_speciation_profiles = /profiles/speciation/aviation/speciati shipping_port_source_pollutants = nox_no2, pm10, pm25, co, so2, nmvoc, ch4, nh3, co2 vessel_list = LC,DC,GC,RO,FE,CR,CO,TU,OT port_list = ACO, ALC, ALI, ALM, ARI, ARR, AVI, ALG, BAR, BIL, CAB, CAD, CSA, CAR, CTG, CAS, CEU, HIE, FER, GAN, GIJ, HUE, IBI, LPM, LCR, MAH, MAL, MPO, MEL, MOT, PMA, PAS, PRO, PSM, SSG, SCP, SCT, SAG, SAL, SCI, SAN, SEV, TAR, TRG, VAL, VIG, VIL, ZFC -hoteling_shapefile_path = /Shapefiles/shipping_port/Areas_Hot_Puertos_and_BCN.shp -maneuvering_shapefile_path = /Shapefiles/shipping_port/Areas_Maneuv_Puertos_and_BCN.shp +hoteling_shapefile_path = /shapefiles/shipping_port/Areas_Hot_Puertos_and_BCN.shp +maneuvering_shapefile_path = /shapefiles/shipping_port/Areas_Maneuv_Puertos_and_BCN.shp shipping_port_ef_path = /shipping_port/ef/engines_fuel_EF.csv shipping_port_engine_percent_path = /shipping_port/ef/ship_perc_engines.csv shipping_port_tonnage_path = /shipping_port/ship_operations_GT_2015.csv @@ -275,10 +275,10 @@ fuel_list = HD_res, LPG_res, NG_res, HD_com, LPG_com, NG_com, B_res, B_com # fuel_list = HD_res, LPG_res, NG_res, HD_com, LPG_com, NG_com residential_source_pollutants = nox_no2, so2, co, nh3, pm10, pm25, nmvoc population_type_map = /jrc/ghsl/original_files/GHS_SMOD_POP2015_GLOBE_R2016A_54009_1k_v1_0.tif -population_type_by_ccaa = /residential/pop_type_ccaa.csv -population_type_by_prov = /residential/pop_type_prov.csv -energy_consumption_by_prov = /residential/energy_consumption_nuts3.csv -energy_consumption_by_ccaa = /residential/energy_consumption_nuts2.csv +population_type_nuts2 = /residential/pop_type_ccaa.csv +population_type_nuts3 = /residential/pop_type_prov.csv +energy_consumption_nuts2 = /residential/energy_consumption_nuts2.csv +energy_consumption_nuts3 = /residential/energy_consumption_nuts3.csv residential_spatial_proxies = /residential/spatial_proxies.csv residential_ef_files_path = /residential/ef/ef.csv residential_heating_degree_day_path = /ecmwf/era5/yearly/heatingdegreeday/hdd_.nc @@ -331,22 +331,22 @@ traffic_hourly_profiles_mean = /profiles/temporal/traffic/aadt_h_mn.c traffic_hourly_profiles_weekday = /profiles/temporal/traffic/aadt_h_wd.csv traffic_hourly_profiles_saturday = /profiles/temporal/traffic/aadt_h_sat.csv traffic_hourly_profiles_sunday = /profiles/temporal/traffic/aadt_h_sun.csv -traffic_speciation_profile_hot_cold = /profiles/speciation/traffic/hot_cold_cmaq_cb05_aero5.csv -traffic_speciation_profile_tyre = /profiles/speciation/traffic/tyre_cmaq_cb05_aero5.csv -traffic_speciation_profile_road = /profiles/speciation/traffic/road_cmaq_cb05_aero5.csv -traffic_speciation_profile_brake = /profiles/speciation/traffic/brake_cmaq_cb05_aero5.csv -traffic_speciation_profile_resuspension = /profiles/speciation/traffic/resuspension_cmaq_cb05_aero5.csv +traffic_speciation_profile_hot_cold = /profiles/speciation/traffic/hot_cold_base.csv +traffic_speciation_profile_tyre = /profiles/speciation/traffic/tyre_base.csv +traffic_speciation_profile_road = /profiles/speciation/traffic/road_base.csv +traffic_speciation_profile_brake = /profiles/speciation/traffic/brake_base.csv +traffic_speciation_profile_resuspension = /profiles/speciation/traffic/resuspension_base.csv [TRAFFIC AREA SECTOR] traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 do_evaporative = 1 traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csv -population_by_municipality = /traffic_area/population_by_mun.csv -traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_cmaq.csv +population_nuts3 = /traffic_area/population_by_mun.csv +traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_base.csv traffic_area_evaporative_ef_file = /traffic_area/ef/evaporative_nmvoc.csv do_small_cities = 1 -traffic_area_small_cities_path = /Shapefiles/small_cities/small_cities.shp -traffic_area_speciation_profiles_small_cities = /profiles/speciation/traffic_area/small_cities_cmaq.csv +traffic_area_small_cities_path = /shapefiles/small_cities/small_cities.shp +traffic_area_speciation_profiles_small_cities = /profiles/speciation/traffic_area/small_cities_base.csv traffic_area_small_cities_ef_file = /traffic_area/ef/small_cities.csv small_cities_monthly_profile = /profiles/temporal/traffic_area/small_cities_monthly_profiles.csv small_cities_weekly_profile = /profiles/temporal/traffic_area/small_cities_weekly_profiles.csv diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 5ed5bd7..fb1eb01 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -167,8 +167,8 @@ class Config(ArgParser): p.add_argument('--speciation_map', required=False, help='...') # ===== SHAPEFILES ===== - p.add_argument('--nut_shapefile_prov', required=False, type=str, default='True') - p.add_argument('--nut_shapefile_ccaa', required=False, type=str, default='True') + p.add_argument('--nuts3_shapefile', required=False, type=str, default='True') + p.add_argument('--nuts2_shapefile', required=False, type=str, default='True') p.add_argument('--clipping', required=False, type=str, default=None, help='To clip the domain into an specific zone. ' + @@ -288,10 +288,10 @@ class Config(ArgParser): p.add_argument('--residential_source_pollutants', required=False, help='...') p.add_argument('--population_density_map', required=False, help='...') p.add_argument('--population_type_map', required=False, help='...') - p.add_argument('--population_type_by_ccaa', required=False, help='...') - p.add_argument('--population_type_by_prov', required=False, help='...') - p.add_argument('--energy_consumption_by_prov', required=False, help='...') - p.add_argument('--energy_consumption_by_ccaa', required=False, help='...') + p.add_argument('--population_type_nuts2', required=False, help='...') + p.add_argument('--population_type_nuts3', required=False, help='...') + p.add_argument('--energy_consumption_nuts3', required=False, help='...') + p.add_argument('--energy_consumption_nuts2', required=False, help='...') p.add_argument('--residential_spatial_proxies', required=False, help='...') p.add_argument('--residential_ef_files_path', required=False, help='...') p.add_argument('--residential_heating_degree_day_path', required=False, help='...') @@ -353,7 +353,7 @@ class Config(ArgParser): p.add_argument('--traffic_area_pollutants', required=False, help='...') p.add_argument('--do_evaporative', required=False, help='...') p.add_argument('--traffic_area_gas_path', required=False, help='...') - p.add_argument('--population_by_municipality', required=False, help='...') + p.add_argument('--population_nuts3', required=False, help='...') p.add_argument('--traffic_area_speciation_profiles_evaporative', required=False, help='...') p.add_argument('--traffic_area_evaporative_ef_file', required=False, help='...') p.add_argument('--do_small_cities', required=False, help='...') diff --git a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py index bb23860..9930f47 100755 --- a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py @@ -31,7 +31,7 @@ class AgriculturalCropOperationsSector(AgriculturalSector): :type date_array: list(datetime.datetime, ...) :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER06' information with the NUT_code. + the 'nuts2_id' information with the NUT_code. :type nut_shapefile_path: str :param source_pollutants: List of input pollutants to take into account. Agricultural livestock module can @@ -84,7 +84,7 @@ class AgriculturalCropOperationsSector(AgriculturalSector): :param crop_by_nut: :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT_code. + the 'nuts3_id' information with the NUT_code. :type nut_shapefile_path: str """ spent_time = timeit.default_timer() diff --git a/hermesv3_bu/sectors/agricultural_machinery_sector.py b/hermesv3_bu/sectors/agricultural_machinery_sector.py index 3dcd17b..d16a18a 100755 --- a/hermesv3_bu/sectors/agricultural_machinery_sector.py +++ b/hermesv3_bu/sectors/agricultural_machinery_sector.py @@ -35,7 +35,7 @@ class AgriculturalMachinerySector(AgriculturalSector): self.crop_machinery_by_nut = self.read_profiles(crop_machinery_by_nut) self.crop_distribution = self.get_crop_distribution_by_nut( - self.crop_distribution, machinery_distibution_nut_shapefile_path, nut_code='ORDER07') + self.crop_distribution, machinery_distibution_nut_shapefile_path, nut_code='nuts3_id') self.months = self.get_date_array_by_month() diff --git a/hermesv3_bu/sectors/agricultural_sector.py b/hermesv3_bu/sectors/agricultural_sector.py index be84980..74e982b 100755 --- a/hermesv3_bu/sectors/agricultural_sector.py +++ b/hermesv3_bu/sectors/agricultural_sector.py @@ -230,8 +230,8 @@ class AgriculturalSector(Sector): land_uses_shp = IoRaster(self.comm_agr).to_shapefile_serie(land_uses_clipped) ccaa_shp = IoShapefile(self.comm_agr).read_shapefile_serial(self.nut_shapefile).to_crs(land_uses_shp.crs) - ccaa_shp.drop(columns=['NAME', 'ORDER06'], inplace=True) - ccaa_shp.rename(columns={'CODE': 'NUT'}, inplace=True) + ccaa_shp.drop(columns=['nuts2_na'], inplace=True) + ccaa_shp.rename(columns={'nuts2_id': 'NUT'}, inplace=True) # ccaa_shp.set_index('NUT', inplace=True) land_use_src_by_nut = self.spatial_overlays(land_uses_shp.reset_index(), ccaa_shp, how='intersection') land_use_src_by_nut.drop(columns=['idx1', 'idx2', 'CELL_ID'], inplace=True) diff --git a/hermesv3_bu/sectors/livestock_sector.py b/hermesv3_bu/sectors/livestock_sector.py index 80d4b35..de099bd 100755 --- a/hermesv3_bu/sectors/livestock_sector.py +++ b/hermesv3_bu/sectors/livestock_sector.py @@ -148,7 +148,7 @@ class LivestockSector(Sector): :type molecular_weights_path: str :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT_code. + the 'nuts3_id' information with the NUT_code. :type nut_shapefile_path: str """ spent_time = timeit.default_timer() @@ -187,7 +187,7 @@ class LivestockSector(Sector): :type gridded_livestock_path: str :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT ID. + the 'nuts3_id' information with the NUT ID. :type nut_shapefile_path: str :param correction_split_factors_path: Path to the CSV file that contains the correction factors and the @@ -420,7 +420,7 @@ class LivestockSector(Sector): :type dataframe: geopandas.GeoDataframe :param nut_shapefile_path: Path to the shapefile that contain the NUT polygons. The shapefile must contain - the 'ORDER07' information with the NUT_code. + the 'nuts3_id' information with the NUT_code. :type nut_shapefile_path: str :param correction_split_factors_path: Path to the CSV file that contains the correction factors and the @@ -444,7 +444,7 @@ class LivestockSector(Sector): 'animal_distribution_by_cat.shp') if not os.path.exists(animal_distribution_path): - dataframe = self.add_nut_code(dataframe, nut_shapefile_path, nut_value='ORDER07') + dataframe = self.add_nut_code(dataframe, nut_shapefile_path, nut_value='nuts3_id') splitting_factors = self.get_splitting_factors(correction_split_factors_path) diff --git a/hermesv3_bu/sectors/residential_sector.py b/hermesv3_bu/sectors/residential_sector.py index 22cd116..495b5e7 100755 --- a/hermesv3_bu/sectors/residential_sector.py +++ b/hermesv3_bu/sectors/residential_sector.py @@ -18,8 +18,8 @@ from hermesv3_bu.logger.log import Log class ResidentialSector(Sector): def __init__(self, comm, logger, auxiliary_dir, grid_shp, clip, date_array, source_pollutants, vertical_levels, fuel_list, prov_shapefile, ccaa_shapefile, population_density_map, population_type_map, - population_type_by_ccaa, population_type_by_prov, energy_consumption_by_prov, - energy_consumption_by_ccaa, residential_spatial_proxies, residential_ef_files_path, + population_type_nuts2, population_type_nuts3, energy_consumption_nuts3, + energy_consumption_nuts2, residential_spatial_proxies, residential_ef_files_path, heating_degree_day_path, temperature_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path): spent_time = timeit.default_timer() @@ -37,11 +37,11 @@ class ResidentialSector(Sector): self.fuel_list = fuel_list self.day_dict = self.calculate_num_days() - self.pop_type_by_prov = population_type_by_prov - self.pop_type_by_ccaa = population_type_by_ccaa + self.pop_type_by_prov = population_type_nuts3 + self.pop_type_by_ccaa = population_type_nuts2 - self.energy_consumption_by_prov = pd.read_csv(energy_consumption_by_prov) - self.energy_consumption_by_ccaa = pd.read_csv(energy_consumption_by_ccaa) + self.energy_consumption_nuts3 = pd.read_csv(energy_consumption_nuts3) + self.energy_consumption_nuts2 = pd.read_csv(energy_consumption_nuts2) self.residential_spatial_proxies = self.read_residential_spatial_proxies(residential_spatial_proxies) self.ef_profiles = self.read_ef_file(residential_ef_files_path) @@ -173,10 +173,10 @@ class ResidentialSector(Sector): # population_density = gpd.sjoin(population_density, population_type, how='left', op='intersects') # population_density.drop(columns=['index_right'], inplace=True) - population_density = self.add_nut_code(population_density, prov_shapefile, nut_value='ORDER07') + population_density = self.add_nut_code(population_density, prov_shapefile, nut_value='nuts3_id') population_density.rename(columns={'nut_code': 'prov'}, inplace=True) population_density = population_density.loc[population_density['prov'] != -999, :] - population_density = self.add_nut_code(population_density, ccaa_shapefile, nut_value='ORDER06') + population_density = self.add_nut_code(population_density, ccaa_shapefile, nut_value='nuts2_id') population_density.rename(columns={'nut_code': 'ccaa'}, inplace=True) population_density = population_density.loc[population_density['ccaa'] != -999, :] @@ -186,8 +186,10 @@ class ResidentialSector(Sector): population_density.loc[:, ['ccaa', 'pop', 'type']].groupby(['ccaa', 'type']).sum().reset_index().to_csv( self.pop_type_by_ccaa) - self.pop_type_by_ccaa = pd.read_csv(self.pop_type_by_ccaa).set_index(['ccaa', 'type']) - self.pop_type_by_prov = pd.read_csv(self.pop_type_by_prov).set_index(['prov', 'type']) + self.pop_type_by_ccaa = pd.read_csv(self.pop_type_by_ccaa).rename( + columns={'nuts2_id': 'ccaa'}).set_index(['ccaa', 'type']) + self.pop_type_by_prov = pd.read_csv(self.pop_type_by_prov).rename( + columns={'nuts3_id': 'prov'}).set_index(['prov', 'type']) fuel_distribution = population_density[['geometry']].copy() fuel_distribution.index.name = 'CELL_ID' @@ -202,8 +204,8 @@ class ResidentialSector(Sector): if spatial_proxy['proxy_type'] == 'all': total_pop = self.pop_type_by_ccaa.loc[ self.pop_type_by_ccaa.index.get_level_values('ccaa') == ccaa, 'pop'].sum() - energy_consumption = self.energy_consumption_by_ccaa.loc[ - self.energy_consumption_by_ccaa['code'] == ccaa, fuel].values[0] + energy_consumption = self.energy_consumption_nuts2.loc[ + self.energy_consumption_nuts2['nuts2_id'] == ccaa, fuel].values[0] fuel_distribution.loc[ population_density['ccaa'] == ccaa, fuel] = population_density['pop'].multiply( @@ -213,8 +215,8 @@ class ResidentialSector(Sector): (self.pop_type_by_ccaa.index.get_level_values('ccaa') == ccaa) & (self.pop_type_by_ccaa.index.get_level_values('type') == spatial_proxy['proxy_type']), 'pop'].values[0] - energy_consumption = self.energy_consumption_by_ccaa.loc[ - self.energy_consumption_by_ccaa['code'] == ccaa, fuel].values[0] + energy_consumption = self.energy_consumption_nuts2.loc[ + self.energy_consumption_nuts2['nuts2_id'] == ccaa, fuel].values[0] fuel_distribution.loc[(population_density['ccaa'] == ccaa) & (population_density['type'] == spatial_proxy['proxy_type']), @@ -225,8 +227,8 @@ class ResidentialSector(Sector): if spatial_proxy['proxy_type'] == 'all': total_pop = self.pop_type_by_prov.loc[self.pop_type_by_prov.index.get_level_values( 'prov') == prov, 'pop'].sum() - energy_consumption = self.energy_consumption_by_prov.loc[ - self.energy_consumption_by_prov['code'] == prov, fuel].values[0] + energy_consumption = self.energy_consumption_nuts3.loc[ + self.energy_consumption_nuts3['nuts3_id'] == prov, fuel].values[0] fuel_distribution.loc[population_density['prov'] == prov, fuel] = population_density[ 'pop'].multiply(energy_consumption / total_pop) @@ -235,21 +237,19 @@ class ResidentialSector(Sector): (self.pop_type_by_prov.index.get_level_values('prov') == prov) & (self.pop_type_by_prov.index.get_level_values('type') == spatial_proxy['proxy_type']), 'pop'].values[0] - energy_consumption = self.energy_consumption_by_prov.loc[ - self.energy_consumption_by_prov['code'] == prov, fuel].values[0] + energy_consumption = self.energy_consumption_nuts3.loc[ + self.energy_consumption_nuts3['nuts3_id'] == prov, fuel].values[0] fuel_distribution.loc[(population_density['prov'] == prov) & (population_density['type'] == spatial_proxy['proxy_type']), fuel] = population_density['pop'].multiply( energy_consumption / total_pop) fuel_distribution = self.to_dst_resolution(fuel_distribution) - + fuel_distribution.set_index('FID', inplace=True) IoShapefile(self.comm).write_shapefile_serial(fuel_distribution.reset_index(), fuel_distribution_path) else: fuel_distribution = IoShapefile(self.comm).read_shapefile_serial(fuel_distribution_path) - fuel_distribution.drop(columns=['index'], inplace=True) - - fuel_distribution.set_index('FID', inplace=True) + fuel_distribution.set_index('FID', inplace=True) self.logger.write_time_log('ResidentialSector', 'get_fuel_distribution', timeit.default_timer() - spent_time) return fuel_distribution diff --git a/hermesv3_bu/sectors/sector.py b/hermesv3_bu/sectors/sector.py index edcf955..8aae0b5 100755 --- a/hermesv3_bu/sectors/sector.py +++ b/hermesv3_bu/sectors/sector.py @@ -421,7 +421,7 @@ class Sector(object): return dataframe - def add_nut_code(self, shapefile, nut_shapefile_path, nut_value='ORDER06'): + def add_nut_code(self, shapefile, nut_shapefile_path, nut_value='nuts2_id'): """ Add 'nut_code' column into the shapefile based on the 'nut_value' column of the 'nut_shapefile_path' shapefile. diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index b306dca..f84ce5d 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -70,7 +70,7 @@ class SectorManager(object): arguments.denominator_yearly_factor_dir, arguments.livestock_ef_files_dir, arguments.livestock_monthly_profiles, arguments.livestock_weekly_profiles, arguments.livestock_hourly_profiles, arguments.speciation_map, - arguments.livestock_speciation_profiles, arguments.molecular_weights, arguments.nut_shapefile_prov) + arguments.livestock_speciation_profiles, arguments.molecular_weights, arguments.nuts3_shapefile) elif sector == 'crop_operations' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.agricultural_crop_operations_sector import AgriculturalCropOperationsSector @@ -80,7 +80,7 @@ class SectorManager(object): self.sector = AgriculturalCropOperationsSector( comm_agr, comm, logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.crop_operations_source_pollutants, - grid.vertical_desctiption, arguments.crop_operations_list, arguments.nut_shapefile_ccaa, + grid.vertical_desctiption, arguments.crop_operations_list, arguments.nuts2_shapefile, arguments.land_uses_path, arguments.crop_operations_ef_files_dir, arguments.crop_operations_monthly_profiles, arguments.crop_operations_weekly_profiles, arguments.crop_operations_hourly_profiles, arguments.speciation_map, @@ -95,7 +95,7 @@ class SectorManager(object): self.sector = AgriculturalCropFertilizersSector( comm_agr, comm, logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.crop_fertilizers_source_pollutants, grid.vertical_desctiption, - arguments.crop_fertilizers_list, arguments.nut_shapefile_ccaa, arguments.land_uses_path, + arguments.crop_fertilizers_list, arguments.nuts2_shapefile, arguments.land_uses_path, arguments.crop_fertilizers_hourly_profiles, arguments.speciation_map, arguments.crop_fertilizers_speciation_profiles, arguments.molecular_weights, arguments.land_use_by_nut_path, arguments.crop_by_nut_path, arguments.crop_from_landuse_path, @@ -113,13 +113,13 @@ class SectorManager(object): self.sector = AgriculturalMachinerySector( comm_agr, comm, logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.crop_machinery_source_pollutants, grid.vertical_desctiption, - arguments.crop_machinery_list, arguments.nut_shapefile_ccaa, arguments.machinery_list, + arguments.crop_machinery_list, arguments.nuts2_shapefile, arguments.machinery_list, arguments.land_uses_path, arguments.crop_machinery_ef_path, arguments.crop_machinery_monthly_profiles, arguments.crop_machinery_weekly_profiles, arguments.crop_machinery_hourly_profiles, arguments.speciation_map, arguments.crop_machinery_speciation_profiles, arguments.molecular_weights, arguments.land_use_by_nut_path, arguments.crop_by_nut_path, arguments.crop_from_landuse_path, - arguments.nut_shapefile_prov, arguments.crop_machinery_deterioration_factor_path, + arguments.nuts3_shapefile, arguments.crop_machinery_deterioration_factor_path, arguments.crop_machinery_load_factor_path, arguments.crop_machinery_vehicle_ratio_path, arguments.crop_machinery_vehicle_units_path, arguments.crop_machinery_vehicle_workhours_path, arguments.crop_machinery_vehicle_power_path, arguments.crop_machinery_by_nut) @@ -130,9 +130,9 @@ class SectorManager(object): comm_world.Split(color, sector_procs.index(comm_world.Get_rank())), self.logger, arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.residential_source_pollutants, grid.vertical_desctiption, arguments.fuel_list, - arguments.nut_shapefile_prov, arguments.nut_shapefile_ccaa, arguments.population_density_map, - arguments.population_type_map, arguments.population_type_by_ccaa, arguments.population_type_by_prov, - arguments.energy_consumption_by_prov, arguments.energy_consumption_by_ccaa, + arguments.nuts3_shapefile, arguments.nuts2_shapefile, arguments.population_density_map, + arguments.population_type_map, arguments.population_type_nuts2, arguments.population_type_nuts3, + arguments.energy_consumption_nuts3, arguments.energy_consumption_nuts2, arguments.residential_spatial_proxies, arguments.residential_ef_files_path, arguments.residential_heating_degree_day_path, arguments.temperature_daily_files_path, arguments.residential_hourly_profiles, arguments.speciation_map, @@ -196,7 +196,7 @@ class SectorManager(object): arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.traffic_area_pollutants, grid.vertical_desctiption, arguments.population_density_map, arguments.speciation_map, arguments.molecular_weights, arguments.do_evaporative, arguments.traffic_area_gas_path, - arguments.population_by_municipality, arguments.nut_shapefile_prov, + arguments.population_nuts3, arguments.nuts3_shapefile, arguments.traffic_area_speciation_profiles_evaporative, arguments.traffic_area_evaporative_ef_file, arguments.temperature_hourly_files_path, arguments.do_small_cities, arguments.traffic_area_small_cities_path, arguments.traffic_area_speciation_profiles_small_cities, diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 75932b4..f870161 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -115,7 +115,7 @@ class TrafficAreaSector(Sector): return population_density def make_population_by_nuts(self, population_shape, nut_shp, pop_by_nut_path, write_file=True, csv_path=None, - column_id='ORDER07'): + column_id='nuts3_id'): spent_time = timeit.default_timer() if not os.path.exists(pop_by_nut_path): @@ -142,8 +142,7 @@ class TrafficAreaSector(Sector): if not os.path.exists(pop_nut_cell_path): pop_by_nut = pop_by_nut.to_crs(self.grid_shp.crs) - - del pop_by_nut['NAME'] + # del pop_by_nut[''] pop_by_nut['area_in'] = pop_by_nut.geometry.area # df = gpd.overlay(pop_by_nut, grid_shp, how='intersection') @@ -162,7 +161,7 @@ class TrafficAreaSector(Sector): return df def make_vehicles_by_cell(self, pop_nut_cell, gasoline_path, total_pop_by_nut, veh_by_cell_path, - column_id='ORDER07'): + column_id='nuts3_id'): spent_time = timeit.default_timer() if not os.path.exists(veh_by_cell_path): @@ -172,8 +171,8 @@ class TrafficAreaSector(Sector): df = pd.merge(pop_nut_cell, total_pop_by_nut, left_on=column_id, right_on=column_id, how='left') - df['pop_percent'] = df['data_x'] / df['data_y'] - df.drop(columns=['data_x', 'data_y'], inplace=True) + df['pop_percent'] = df['data'] / df['population'] + df.drop(columns=['data', 'population'], inplace=True) gas_df = pd.read_csv(gasoline_path, index_col='COPERT_V_name').transpose() vehicle_type_list = list(gas_df.columns.values) -- GitLab From 8b16b1d420a08e31bc208f243db52534245a4084 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 23 Aug 2019 09:51:46 +0200 Subject: [PATCH 2/6] LIvestock: NUT -> nuts3_na, nuts3_id --- conf/hermes.conf | 10 +++++----- hermesv3_bu/sectors/livestock_sector.py | 21 +++++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 7a0943c..6b501e0 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,14 +3,14 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__res.nc +output_name = HERMESv3__live.nc emission_summary = 1 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 output_timestep_num = 25 auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ -erase_auxiliary_files = 0 +erase_auxiliary_files = 1 [DOMAIN] @@ -122,7 +122,7 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver # SECTORS #################################################################### [SECTOR MANAGEMENT] -writing_processors = 12 +writing_processors = 1 # aviation_processors = 1 # shipping_port_processors = 1 @@ -138,11 +138,11 @@ writing_processors = 12 aviation_processors = 0 shipping_port_processors = 0 -livestock_processors = 0 +livestock_processors = 1 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 -residential_processors = 1 +residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 traffic_processors = 0 diff --git a/hermesv3_bu/sectors/livestock_sector.py b/hermesv3_bu/sectors/livestock_sector.py index de099bd..305dabd 100755 --- a/hermesv3_bu/sectors/livestock_sector.py +++ b/hermesv3_bu/sectors/livestock_sector.py @@ -194,7 +194,7 @@ class LivestockSector(Sector): splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["NUT", "nut_code", "_fact", "_01", ...] + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01", ...] "nut_code" column must contain the NUT ID. :type correction_split_factors_path: str @@ -383,7 +383,7 @@ class LivestockSector(Sector): splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["NUT", "nut_code", "_fact", "_01", ...] + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01", ...] "nut_code" column must contain the NUT ID. :type correction_split_factors_path: str @@ -394,20 +394,20 @@ class LivestockSector(Sector): splitting_factors_list = [] for animal in self.animal_list: correction_split_factors = pd.read_csv(correction_split_factors_path.replace('', animal)) - correction_split_factors.set_index('nut_code', inplace=True) + correction_split_factors.set_index('nuts3_id', inplace=True) categories = list(correction_split_factors.columns.values) - categories = [e for e in categories if e not in ['NUT', 'nut_code', '{0}_fact'.format(animal)]] + categories = [e for e in categories if e not in ['nuts3_na', 'nuts3_id', '{0}_fact'.format(animal)]] correction_split_factors[categories] = correction_split_factors.loc[:, categories].multiply( correction_split_factors['{0}_fact'.format(animal)], axis='index') - correction_split_factors.drop(columns=['NUT', '{0}_fact'.format(animal)], inplace=True) + correction_split_factors.drop(columns=['nuts3_na', '{0}_fact'.format(animal)], inplace=True) splitting_factors_list.append(correction_split_factors) splitting_factors = pd.concat(splitting_factors_list, axis=1) splitting_factors.reset_index(inplace=True) - splitting_factors['nut_code'] = splitting_factors['nut_code'].astype(np.int16) + splitting_factors['nuts3_id'] = splitting_factors['nuts3_id'].astype(np.int16) self.logger.write_time_log('LivestockSector', 'get_splitting_factors', timeit.default_timer() - spent_time) return splitting_factors @@ -426,9 +426,9 @@ class LivestockSector(Sector): :param correction_split_factors_path: Path to the CSV file that contains the correction factors and the splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["NUT", "nut_code", "_fact", "_01", + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01", ...] - "nut_code" column must contain the NUT ID + "nuts3_id" column must contain the NUT ID :type correction_split_factors_path: str :return: GeoDataframe with the amount of each animal subtype by destiny cell (FID) @@ -445,13 +445,14 @@ class LivestockSector(Sector): if not os.path.exists(animal_distribution_path): dataframe = self.add_nut_code(dataframe, nut_shapefile_path, nut_value='nuts3_id') + dataframe.rename(columns={'nut_code': 'nuts3_id'}, inplace=True) splitting_factors = self.get_splitting_factors(correction_split_factors_path) # Adding the splitting factors by NUT code - dataframe = pd.merge(dataframe, splitting_factors, how='left', on='nut_code') + dataframe = pd.merge(dataframe, splitting_factors, how='left', on='nuts3_id') - dataframe.drop(columns=['nut_code'], inplace=True) + dataframe.drop(columns=['nuts3_id'], inplace=True) for animal in self.animal_list: animal_types = [i for i in list(dataframe.columns.values) if i.startswith(animal)] -- GitLab From e59d60982319f4be58550a6eb4c7e9f919c88134 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 23 Aug 2019 13:48:42 +0200 Subject: [PATCH 3/6] Crop operations: NUT -> nuts2_na, nuts2_id --- hermesv3_bu/sectors/agricultural_sector.py | 27 +++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/hermesv3_bu/sectors/agricultural_sector.py b/hermesv3_bu/sectors/agricultural_sector.py index 74e982b..00f1973 100755 --- a/hermesv3_bu/sectors/agricultural_sector.py +++ b/hermesv3_bu/sectors/agricultural_sector.py @@ -258,7 +258,8 @@ class AgriculturalSector(Sector): """ spent_time = timeit.default_timer() - df = pd.read_csv(self.land_use_by_nut) + df = pd.read_csv(self.land_use_by_nut, dtype={'nuts2_id': str}) + df.rename(columns={'nuts2_id': 'NUT'}, inplace=True) df = df.loc[df['land_use'].isin(land_uses), :] df.set_index(['NUT', 'land_use'], inplace=True) @@ -280,11 +281,12 @@ class AgriculturalSector(Sector): """ spent_time = timeit.default_timer() - land_use_distribution_src_nut['area'] = land_use_distribution_src_nut.area - land_use_by_nut = land_use_distribution_src_nut.groupby(['NUT', 'land_use']).sum().reset_index() - land_use_by_nut = land_use_by_nut.loc[land_use_by_nut['land_use'].isin(land_uses), :] + land_use_by_nut = pd.DataFrame(index=pd.MultiIndex.from_product( + [np.unique(land_use_distribution_src_nut['NUT']), land_uses], names=['NUT', 'land_use'])) - land_use_by_nut.set_index(['NUT', 'land_use'], inplace=True) + land_use_distribution_src_nut['area'] = land_use_distribution_src_nut.area + land_use_by_nut['area'] = land_use_distribution_src_nut.groupby(['NUT', 'land_use']).sum() + land_use_by_nut.fillna(0.0, inplace=True) self.logger.write_time_log('AgriculturalSector', 'get_land_use_by_nut_csv', timeit.default_timer() - spent_time) return land_use_by_nut @@ -303,9 +305,9 @@ class AgriculturalSector(Sector): :rtype: DataFrame """ spent_time = timeit.default_timer() - if nuts is not None: - land_use_by_nut = land_use_by_nut.loc[nuts, :] + land_use_by_nut = land_use_by_nut.iloc[land_use_by_nut.index.get_level_values('NUT').isin(nuts)] + new_df = pd.DataFrame(index=np.unique(land_use_by_nut.index.get_level_values('NUT')), columns=self.crop_from_landuse.keys()) new_df.fillna(0, inplace=True) @@ -336,6 +338,7 @@ class AgriculturalSector(Sector): crop_share_by_nut = crop_by_nut.copy() for crop in crop_by_nut.columns: crop_share_by_nut[crop] = crop_by_nut[crop] / tot_crop_by_nut[crop] + self.logger.write_time_log('AgriculturalSector', 'get_crop_shape_by_nut', timeit.default_timer() - spent_time) return crop_share_by_nut @@ -353,11 +356,13 @@ class AgriculturalSector(Sector): """ spent_time = timeit.default_timer() - crop_by_nut = pd.read_csv(self.crop_by_nut) - crop_by_nut.drop(columns='name', inplace=True) + crop_by_nut = pd.read_csv(self.crop_by_nut, dtype={'nuts2_id': str}) + crop_by_nut.drop(columns='nuts2_na', inplace=True) + crop_by_nut.rename(columns={'nuts2_id': 'code'}, inplace=True) - crop_by_nut['code'] = crop_by_nut['code'].astype(np.int16) + # crop_by_nut['code'] = crop_by_nut['code'].astype(np.int16) crop_by_nut.set_index('code', inplace=True) + crop_by_nut = crop_by_nut.loc[crop_share_by_nut.index, :] crop_area_by_nut = crop_share_by_nut * crop_by_nut @@ -455,7 +460,6 @@ class AgriculturalSector(Sector): message_level=2) involved_land_uses = self.get_involved_land_uses() - land_use_distribution_src_nut = self.get_land_use_src_by_nut(involved_land_uses) land_use_by_nut = self.get_land_use_by_nut_csv(land_use_distribution_src_nut, involved_land_uses) tot_land_use_by_nut = self.get_tot_land_use_by_nut(involved_land_uses) @@ -463,6 +467,7 @@ class AgriculturalSector(Sector): crop_by_nut = self.land_use_to_crop_by_nut(land_use_by_nut) tot_crop_by_nut = self.land_use_to_crop_by_nut( tot_land_use_by_nut, nuts=list(np.unique(land_use_by_nut.index.get_level_values('NUT')))) + crop_shape_by_nut = self.get_crop_shape_by_nut(crop_by_nut, tot_crop_by_nut) crop_area_by_nut = self.get_crop_area_by_nut(crop_shape_by_nut) -- GitLab From f41af4018428bab181726d4bceaa551d5c39d6a5 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 23 Aug 2019 16:05:04 +0200 Subject: [PATCH 4/6] -Point sources: Fixed flip from netCDF X Y --- hermesv3_bu/sectors/point_source_sector.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index 0346340..a13d765 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -440,7 +440,7 @@ class PointSourceSector(Sector): var = netcdf.variables[var_name][time_index, 0, :] netcdf.close() - dataframe[var_name] = var[dataframe['X'], dataframe['Y']] + dataframe[var_name] = var[dataframe['Y'], dataframe['X']] return dataframe[[var_name]] @@ -457,7 +457,7 @@ class PointSourceSector(Sector): var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] pre_t_lay = 0 lay_list = [] @@ -488,7 +488,7 @@ class PointSourceSector(Sector): var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] lay_list = ['temp_sfc'] for i, t_lay in enumerate(var): @@ -521,7 +521,7 @@ class PointSourceSector(Sector): var = u10_netcdf.variables[u_var_name][time_index, 0, :] u10_netcdf.close() - dataframe['u10'] = var[dataframe['X'], dataframe['Y']] + dataframe['u10'] = var[dataframe['Y'], dataframe['X']] # === v10 === v10_nc_path = os.path.join( @@ -530,7 +530,7 @@ class PointSourceSector(Sector): var = v10_netcdf.variables[v_var_name][time_index, 0, :] v10_netcdf.close() - dataframe['v10'] = var[dataframe['X'], dataframe['Y']] + dataframe['v10'] = var[dataframe['Y'], dataframe['X']] # === wind speed === dataframe['wSpeed_10'] = np.linalg.norm(dataframe[['u10', 'v10']].values, axis=1) @@ -552,7 +552,7 @@ class PointSourceSector(Sector): var = np.flipud(u10_netcdf.variables[u_var_name][time_index, :, :, :]) u10_netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] for i, t_lay in enumerate(var): dataframe['u_{0}'.format(i)] = t_lay @@ -564,7 +564,7 @@ class PointSourceSector(Sector): var = np.flipud(v10_netcdf.variables[v_var_name][time_index, :, :, :]) v10_netcdf.close() - var = var[:, dataframe['X'], dataframe['Y']] + var = var[:, dataframe['Y'], dataframe['X']] ws_lay_list = ['wSpeed_10'] for i, t_lay in enumerate(var): -- GitLab From e8fcaa9560402ebf3a04a6343f02f3cc19657e4c Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 23 Aug 2019 16:39:39 +0200 Subject: [PATCH 5/6] -Agricultural: nomenclature conventions --- conf/hermes.conf | 16 +++++++++------- hermesv3_bu/config/config.py | 2 +- .../agricultural_crop_fertilizers_sector.py | 2 ++ .../sectors/agricultural_machinery_sector.py | 19 ++++++++++--------- hermesv3_bu/sectors/sector_manager.py | 2 +- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 6b501e0..a721eef 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,12 +3,12 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__live.nc -emission_summary = 1 +output_name = HERMESv3__crop_fert.nc +emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 -output_timestep_num = 25 +output_timestep_num = 24 auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ erase_auxiliary_files = 1 @@ -65,6 +65,8 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver # CATALUNYA test nx = 28 ny = 30 + #nx = 4 + #ny = 4 inc_x = 10000 inc_y = 10000 x_0 = 253151.59375 @@ -138,9 +140,9 @@ writing_processors = 1 aviation_processors = 0 shipping_port_processors = 0 -livestock_processors = 1 +livestock_processors = 0 crop_operations_processors = 0 -crop_fertilizers_processors = 0 +crop_fertilizers_processors = 1 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 @@ -267,7 +269,7 @@ crop_machinery_monthly_profiles = /profiles/temporal/agricultural_mac crop_machinery_weekly_profiles = /profiles/temporal/agricultural_machinery/weekly_profiles.csv crop_machinery_hourly_profiles = /profiles/temporal/agricultural_machinery/hourly_profiles.csv crop_machinery_speciation_profiles = /profiles/speciation/agricultural_machinery/speciation_profiles_base.csv -crop_machinery_by_nut = /agriculture/agricultural_machinery/crops_ha_prov_2017.csv +crop_machinery_nuts3 = /agriculture/agricultural_machinery/crops_ha_prov_2017.csv [RESIDENTIAL] fuel_list = HD_res, LPG_res, NG_res, HD_com, LPG_com, NG_com, B_res, B_com @@ -298,7 +300,7 @@ recreational_boats_speciation_profiles = /profiles/speciation/recreat [POINT SOURCES] point_source_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25,ch4,n2o,co2 -plume_rise = False +plume_rise = True # point_source_snaps = 09 point_source_catalog = /point_sources/Maestra_focos_2015_plume_rise.shp point_source_monthly_profiles = /profiles/temporal/point_sources/monthly_profiles.csv diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index fb1eb01..3c9b6bd 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -281,7 +281,7 @@ class Config(ArgParser): p.add_argument('--crop_machinery_hourly_profiles', required=False, help='...') p.add_argument('--crop_machinery_speciation_map', required=False, help='...') p.add_argument('--crop_machinery_speciation_profiles', required=False, help='...') - p.add_argument('--crop_machinery_by_nut', required=False, help='...') + p.add_argument('--crop_machinery_nuts3', required=False, help='...') # ***** RESIDENTIAL SECTOR ***** p.add_argument('--fuel_list', required=False, help='...') diff --git a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py index 25aeec0..924d70d 100755 --- a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py @@ -32,7 +32,9 @@ class AgriculturalCropFertilizersSector(AgriculturalSector): self.cultivated_ratio = self.read_profiles(cultivated_ratio) self.fertilizer_rate = self.read_profiles(fertilizer_rate) + self.fertilizer_rate.rename(columns={'nuts2_id': 'code'}, inplace=True) self.crop_f_parameter = self.read_profiles(crop_f_parameter) + self.crop_f_parameter.rename(columns={'nuts2_id': 'code'}, inplace=True) self.crop_f_fertilizers = self.read_profiles(crop_f_fertilizers) if self.comm.Get_rank() == 0: diff --git a/hermesv3_bu/sectors/agricultural_machinery_sector.py b/hermesv3_bu/sectors/agricultural_machinery_sector.py index d16a18a..806c5da 100755 --- a/hermesv3_bu/sectors/agricultural_machinery_sector.py +++ b/hermesv3_bu/sectors/agricultural_machinery_sector.py @@ -21,7 +21,7 @@ class AgriculturalMachinerySector(AgriculturalSector): speciation_profiles_path, molecular_weights_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, machinery_distibution_nut_shapefile_path, deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, vehicle_workhours_path, vehicle_power_path, - crop_machinery_by_nut): + crop_machinery_nuts3): spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL MACHINERY SECTOR =====') @@ -32,7 +32,7 @@ class AgriculturalMachinerySector(AgriculturalSector): speciation_profiles_path, molecular_weights_path) self.machinery_list = machinery_list - self.crop_machinery_by_nut = self.read_profiles(crop_machinery_by_nut) + self.crop_machinery_nuts3 = self.read_profiles(crop_machinery_nuts3) self.crop_distribution = self.get_crop_distribution_by_nut( self.crop_distribution, machinery_distibution_nut_shapefile_path, nut_code='nuts3_id') @@ -53,7 +53,7 @@ class AgriculturalMachinerySector(AgriculturalSector): spent_time = timeit.default_timer() def get_fraction(dataframe): - total_crop_sum = self.crop_machinery_by_nut.loc[self.crop_machinery_by_nut[nut_code] == int(dataframe.name), + total_crop_sum = self.crop_machinery_nuts3.loc[self.crop_machinery_nuts3[nut_code] == int(dataframe.name), self.crop_list].values.sum() dataframe['fraction'] = dataframe[self.crop_list].sum(axis=1) / total_crop_sum @@ -81,7 +81,7 @@ class AgriculturalMachinerySector(AgriculturalSector): if write_crop_by_nut: crop_distribution.loc[:, self.crop_list + [nut_code]].groupby(nut_code).sum().reset_index().to_csv( - self.crop_machinery_by_nut) + self.crop_machinery_nuts3) crop_distribution['fraction'] = crop_distribution.groupby(nut_code).apply(get_fraction) crop_distribution.drop(columns=self.crop_list, inplace=True) crop_distribution.rename(columns={nut_code: 'NUT_code'}, inplace=True) @@ -112,30 +112,31 @@ class AgriculturalMachinerySector(AgriculturalSector): spent_time = timeit.default_timer() def get_n(df): - df['N'] = self.vehicle_units.loc[self.vehicle_units['code'] == df.name[0], df.name[1]].values[0] + df['N'] = self.vehicle_units.loc[self.vehicle_units['nuts3_id'] == df.name[0], df.name[1]].values[0] return df.loc[:, ['N']] def get_s(df): df['S'] = self.vehicle_ratio.loc[ - (self.vehicle_ratio['code'] == df.name[0]) & (self.vehicle_ratio['technology'] == df.name[2]), + (self.vehicle_ratio['nuts3_id'] == df.name[0]) & (self.vehicle_ratio['technology'] == df.name[2]), df.name[1]].values[0] return df.loc[:, ['S']] def get_t(df): try: - df['T'] = self.vehicle_workhours.loc[(self.vehicle_workhours['code'] == df.name[0]) & + df['T'] = self.vehicle_workhours.loc[(self.vehicle_workhours['nuts3_id'] == df.name[0]) & (self.vehicle_workhours['technology'] == df.name[2]), df.name[1]].values[0] except IndexError: df['T'] = np.nan df.loc[df['T'].isna(), 'T'] = self.vehicle_workhours.loc[ - (self.vehicle_workhours['code'] == df.name[0]) & (self.vehicle_workhours['technology'] == 'default'), + (self.vehicle_workhours['nuts3_id'] == df.name[0]) & (self.vehicle_workhours['technology'] == + 'default'), df.name[1]].values[0] return df.loc[:, ['T']] def get_p(df): - df['P'] = self.vehicle_power.loc[self.vehicle_power['code'] == df.name[0], df.name[1]].values[0] + df['P'] = self.vehicle_power.loc[self.vehicle_power['nuts3_id'] == df.name[0], df.name[1]].values[0] return df.loc[:, ['P']] def get_lf(df): diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index f84ce5d..048eeb6 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -122,7 +122,7 @@ class SectorManager(object): arguments.nuts3_shapefile, arguments.crop_machinery_deterioration_factor_path, arguments.crop_machinery_load_factor_path, arguments.crop_machinery_vehicle_ratio_path, arguments.crop_machinery_vehicle_units_path, arguments.crop_machinery_vehicle_workhours_path, - arguments.crop_machinery_vehicle_power_path, arguments.crop_machinery_by_nut) + arguments.crop_machinery_vehicle_power_path, arguments.crop_machinery_nuts3) elif sector == 'residential' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.residential_sector import ResidentialSector -- GitLab From 07c42881b4453dfc242f14a00c1f037e0db8bbea Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 23 Aug 2019 16:46:41 +0200 Subject: [PATCH 6/6] PEP8 corrections --- hermesv3_bu/sectors/agricultural_machinery_sector.py | 2 +- hermesv3_bu/sectors/livestock_sector.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hermesv3_bu/sectors/agricultural_machinery_sector.py b/hermesv3_bu/sectors/agricultural_machinery_sector.py index 806c5da..12a7be3 100755 --- a/hermesv3_bu/sectors/agricultural_machinery_sector.py +++ b/hermesv3_bu/sectors/agricultural_machinery_sector.py @@ -54,7 +54,7 @@ class AgriculturalMachinerySector(AgriculturalSector): def get_fraction(dataframe): total_crop_sum = self.crop_machinery_nuts3.loc[self.crop_machinery_nuts3[nut_code] == int(dataframe.name), - self.crop_list].values.sum() + self.crop_list].values.sum() dataframe['fraction'] = dataframe[self.crop_list].sum(axis=1) / total_crop_sum return dataframe.loc[:, ['fraction']] diff --git a/hermesv3_bu/sectors/livestock_sector.py b/hermesv3_bu/sectors/livestock_sector.py index 305dabd..8bf264d 100755 --- a/hermesv3_bu/sectors/livestock_sector.py +++ b/hermesv3_bu/sectors/livestock_sector.py @@ -194,7 +194,7 @@ class LivestockSector(Sector): splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01", ...] + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01",...] "nut_code" column must contain the NUT ID. :type correction_split_factors_path: str @@ -383,7 +383,7 @@ class LivestockSector(Sector): splitting factors to discretizise each animal into theirs different animal types. '' will be replaced by each animal of the animal list. - The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01", ...] + The CSV file must contain the following columns ["nuts3_na", "nuts3_id", "_fact", "_01",...] "nut_code" column must contain the NUT ID. :type correction_split_factors_path: str -- GitLab