diff --git a/conf/hermes.conf b/conf/hermes.conf index f3df9174f07d323e7298c3a607169687a2102964..5ebfb165169dac65d17597ca33e7139353ed7334 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__rot.nc +output_name = HERMESv3__point_new.nc emission_summary = 1 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -16,7 +16,7 @@ erase_auxiliary_files = 0 [DOMAIN] # domain_type=[lcc, rotated, mercator, regular] -domain_type = rotated +domain_type = lcc # output_type=[MONARCH, CMAQ, WRF_CHEM, DEFAULT] output_model = DEFAULT output_attributes = /writing/global_attributes_WRF-Chem.csv @@ -144,8 +144,8 @@ crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 -point_sources_processors = 0 -traffic_processors = 1 +point_sources_processors = 1 +traffic_processors = 0 traffic_area_processors = 0 @@ -298,9 +298,9 @@ recreational_boats_speciation_profiles = /profiles/speciation/recreat [POINT SOURCES] point_source_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25,ch4,n2o,co2 -plume_rise = True +plume_rise = False # point_source_snaps = 09 -point_source_catalog = /point_sources/Maestra_Focos_SNAP01030409_2015_plume_rise.csv +point_source_catalog = /point_sources/Maestra_focos_2015_plume_rise.shp point_source_monthly_profiles = /profiles/temporal/point_sources/monthly_profiles.csv point_source_weekly_profiles = /profiles/temporal/point_sources/weekly_profiles.csv point_source_hourly_profiles = /profiles/temporal/point_sources/hourly_profiles.csv diff --git a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py index c7eb22996211460c35aea178916b013edaf011c9..25aeec0db276c7f907f17d8dea1755b3d9612f32 100755 --- a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py @@ -40,9 +40,9 @@ class AgriculturalCropFertilizersSector(AgriculturalSector): self.gridded_constants = self.get_gridded_constants( os.path.join(auxiliary_dir, 'fertilizers', 'gridded_constants.shp'), gridded_ph, - os.path.join(auxiliary_dir, 'fertilizers', 'gridded_ph.tiff'), + os.path.join(auxiliary_dir, 'fertilizers', 'gridded_ph.tif'), gridded_cec, - os.path.join(auxiliary_dir, 'fertilizers', 'gridded_cec.tiff')) + os.path.join(auxiliary_dir, 'fertilizers', 'gridded_cec.tif')) self.ef_by_crop = self.get_ef_by_crop() else: self.logger.write_log('Waiting for master to get the gridded constants', message_level=2) diff --git a/hermesv3_bu/sectors/livestock_sector.py b/hermesv3_bu/sectors/livestock_sector.py index a962101c0f67670b186e334484edeb2c4b72d176..80d4b35a50f0d51e1cd843a59ef5a369a00d3895 100755 --- a/hermesv3_bu/sectors/livestock_sector.py +++ b/hermesv3_bu/sectors/livestock_sector.py @@ -253,7 +253,7 @@ class LivestockSector(Sector): /livestock/animal_distribution//.shp Will be created also the clipped raster (TIFF) following the example path - /livestock/animal_distribution//_clip.tiff + /livestock/animal_distribution//_clip.tif :param gridded_livestock_path: Path to the Raster (TIFF) that contains the animal distribution. '' will be replaced by each animal of the animal list. @@ -276,7 +276,7 @@ class LivestockSector(Sector): clipped_raster_path = IoRaster(self.comm).clip_raster_with_shapefile_poly( gridded_livestock_path.replace('', animal), self.clip.shapefile, os.path.join(self.auxiliary_dir, 'livestock', 'animal_distribution', animal, - '{0}_clip.tiff'.format(animal))) + '{0}_clip.tif'.format(animal))) animal_df = IoRaster(self.comm).to_shapefile_serie(clipped_raster_path, animal_distribution_path, write=True) @@ -401,6 +401,7 @@ class LivestockSector(Sector): correction_split_factors[categories] = correction_split_factors.loc[:, categories].multiply( correction_split_factors['{0}_fact'.format(animal)], axis='index') + correction_split_factors.drop(columns=['NUT', '{0}_fact'.format(animal)], inplace=True) splitting_factors_list.append(correction_split_factors) splitting_factors = pd.concat(splitting_factors_list, axis=1) diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index 4ad98bc09f66362908fa19216dd8c304eac5b11a..a57aa7791df79c39d2b92962e5f4da4ad5e93432 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -58,7 +58,7 @@ class PointSourceSector(Sector): speciation_profiles_path, molecular_weights_path) self.plume_rise = plume_rise - self.catalog = self.read_catalog(catalog_path, sector_list) + self.catalog = self.read_catalog_shapefile(catalog_path, sector_list) self.catalog_measured = self.read_catalog_for_measured_emissions(catalog_path, sector_list) self.measured_path = measured_emission_path @@ -66,7 +66,7 @@ class PointSourceSector(Sector): self.logger.write_time_log('PointSourceSector', '__init__', timeit.default_timer() - spent_time) - def read_catalog(self, catalog_path, sector_list): + def read_catalog_csv(self, catalog_path, sector_list): """ Read the catalog @@ -121,7 +121,64 @@ class PointSourceSector(Sector): self.logger.write_time_log('PointSourceSector', 'read_catalog', timeit.default_timer() - spent_time) return catalog_df - def read_catalog_for_measured_emissions(self, catalog_path, sector_list): + def read_catalog_shapefile(self, catalog_path, sector_list): + """ + Read the catalog + + :param catalog_path: path to the catalog + :type catalog_path: str + + :param sector_list: List of sectors to take into account + :type sector_list: list + + :return: catalog + :rtype: DataFrame + """ + spent_time = timeit.default_timer() + + if self.comm.Get_rank() == 0: + if self.plume_rise: + columns = {"Code": np.str, "Cons": np.bool, "SNAP": np.str, "Height": np.float64, + "Diameter": np.float64, "Speed": np.float64, "Temp": np.float64, "AF": np.float64, + "P_month": np.str, "P_week": np.str, "P_hour": np.str, "P_spec": np.str} + else: + columns = {"Code": np.str, "Cons": np.bool, "SNAP": np.str, "Height": np.float64, "AF": np.float64, + "P_month": np.str, "P_week": np.str, "P_hour": np.str, "P_spec": np.str} + for pollutant in self.source_pollutants: + # EF in Kg / Activity factor + columns['EF_{0}'.format(pollutant)] = np.float64 + + catalog_df = gpd.read_file(catalog_path) + + columns_to_drop = list(set(catalog_df.columns.values) - set(list(columns.keys()) + ['geometry'])) + + if len(columns_to_drop) > 0: + catalog_df.drop(columns=columns_to_drop, inplace=True) + for col, typ in columns.items(): + catalog_df[col] = catalog_df[col].astype(typ) + + # Filtering + catalog_df = catalog_df.loc[catalog_df['Cons'] == 1, :] + catalog_df.drop('Cons', axis=1, inplace=True) + + # Filtering + catalog_df = catalog_df.loc[catalog_df['AF'] != -1, :] + + if sector_list is not None: + catalog_df = catalog_df.loc[catalog_df['SNAP'].str[:2].isin(sector_list)] + catalog_df.drop('SNAP', axis=1, inplace=True) + + catalog_df = gpd.sjoin(catalog_df, self.clip.shapefile.to_crs(catalog_df.crs), how='inner') + catalog_df.drop(columns=['index_right'], inplace=True) + + else: + catalog_df = None + self.comm.Barrier() + catalog_df = IoShapefile(self.comm).split_shapefile(catalog_df) + self.logger.write_time_log('PointSourceSector', 'read_catalog', timeit.default_timer() - spent_time) + return catalog_df + + def read_catalog_for_measured_emissions_csv(self, catalog_path, sector_list): """ Read the catalog @@ -164,6 +221,56 @@ class PointSourceSector(Sector): timeit.default_timer() - spent_time) return catalog_df + def read_catalog_for_measured_emissions(self, catalog_path, sector_list): + """ + Read the catalog + + :param catalog_path: path to the catalog + :type catalog_path: str + + :param sector_list: List of sectors to take into account + :type sector_list: list + + :return: catalog + :rtype: DataFrame + """ + spent_time = timeit.default_timer() + + if self.plume_rise: + columns = {"Code": np.str, "Cons": np.bool, "SNAP": np.str, "Lon": np.float64, "Lat": np.float64, + "Height": np.float64, "Diameter": np.float64, "Speed": np.float64, "Temp": np.float64, + "AF": np.float64, "P_spec": np.str} + else: + columns = {"Code": np.str, "Cons": np.bool, "SNAP": np.str, "Lon": np.float64, "Lat": np.float64, + "Height": np.float64, "AF": np.float64, "P_spec": np.str} + # for pollutant in self.pollutant_list: + # columns['EF_{0}'.format(pollutant)] = settings.precision + + catalog_df = gpd.read_file(catalog_path) + + columns_to_drop = list(set(catalog_df.columns.values) - set(list(columns.keys()) + ['geometry'])) + + if len(columns_to_drop) > 0: + catalog_df.drop(columns=columns_to_drop, inplace=True) + for col, typ in columns.items(): + catalog_df[col] = catalog_df[col].astype(typ) + + # Filtering + catalog_df = catalog_df.loc[catalog_df['Cons'] == 1, :] + catalog_df.drop('Cons', axis=1, inplace=True) + + # Filtering + catalog_df = catalog_df.loc[catalog_df['AF'] == -1, :] + catalog_df.drop('AF', axis=1, inplace=True) + + if sector_list is not None: + catalog_df = catalog_df.loc[catalog_df['SNAP'].str[:2].isin(sector_list)] + catalog_df.drop('SNAP', axis=1, inplace=True) + + self.logger.write_time_log('PointSourceSector', 'read_catalog_for_measured_emissions', + timeit.default_timer() - spent_time) + return catalog_df + def to_geodataframe(self, catalog): """ Convert a simple DataFrame with Lat, Lon columns into a GeoDataFrame as a shape diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index aa2a32ca1c3a91d725403bd3a2d644caf0693816..75932b452578cb3ba9ed51fe811900e560b04b2c 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -15,7 +15,7 @@ pmc_list = ['pmc', 'PMC'] class TrafficAreaSector(Sector): def __init__(self, comm, logger, auxiliary_dir, grid_shp, clip, date_array, source_pollutants, vertical_levels, - population_tiff_path, speciation_map_path, molecular_weights_path, + population_tif_path, speciation_map_path, molecular_weights_path, do_evaporative, gasoline_path, total_pop_by_prov, nuts_shapefile, speciation_profiles_evaporative, evaporative_ef_file, temperature_dir, do_small_cities, small_cities_shp, speciation_profiles_small_cities, small_cities_ef_file, @@ -32,7 +32,7 @@ class TrafficAreaSector(Sector): self.speciation_profiles_evaporative = self.read_speciation_profiles(speciation_profiles_evaporative) self.evaporative_ef_file = evaporative_ef_file if do_evaporative: - self.evaporative = self.init_evaporative(population_tiff_path, nuts_shapefile, gasoline_path, + self.evaporative = self.init_evaporative(population_tif_path, nuts_shapefile, gasoline_path, total_pop_by_prov) else: self.evaporative = None @@ -44,7 +44,7 @@ class TrafficAreaSector(Sector): self.small_cities_weekly_profile = self.read_weekly_profiles(small_cities_weekly_profile) self.small_cities_hourly_profile = self.read_hourly_profiles(small_cities_hourly_profile) if do_small_cities: - self.small_cities = self.init_small_cities(population_tiff_path, small_cities_shp) + self.small_cities = self.init_small_cities(population_tif_path, small_cities_shp) else: self.small_cities = None