diff --git a/conf/hermes.conf b/conf/hermes.conf index 085895749dd2c78c6ac18b1f86bbf7b988a38526..3f70257d0dc732a372012115cda4b9a1f76c928a 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -2,14 +2,14 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon -output_dir = /scratch/Earth/HERMESv3_BU_OUT/parallel -output_name = HERMESv3__residential.nc +output_dir = /scratch/Earth/HERMESv3_BU_OUT +output_name = HERMESv3__tratra.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 output_timestep_num = 24 -auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/parallel/_ +auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ erase_auxiliary_files = 0 @@ -136,10 +136,10 @@ livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 -residential_processors = 2 +residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 -traffic_processors = 0 +traffic_processors = 1 traffic_area_processors = 0 diff --git a/hermesv3_bu/io_server/io_netcdf.py b/hermesv3_bu/io_server/io_netcdf.py index 01d0e837aeeaa354a8f58df00b50f2392a2b4a1a..6c8480d1d6ee1aa33a4ffb4ecfd0208b4d439999 100755 --- a/hermesv3_bu/io_server/io_netcdf.py +++ b/hermesv3_bu/io_server/io_netcdf.py @@ -4,15 +4,16 @@ import sys import os from mpi4py import MPI from datetime import timedelta -from hermesv3_bu.io_server.io_server import IoServer import numpy as np import geopandas as gpd from netCDF4 import Dataset from shapely.geometry import Point from cf_units import num2date, CALENDAR_STANDARD - from geopandas import GeoDataFrame +from hermesv3_bu.io_server.io_server import IoServer +from hermesv3_bu.tools.checker import check_files + class IoNetcdf(IoServer): def __init__(self, comm): @@ -42,6 +43,7 @@ class IoNetcdf(IoServer): :return: GeoDataframe with the data in the desired points. :rtype: geopandas.GeoDataframe """ + check_files(netcdf_path) nc = Dataset(netcdf_path, mode='r') lat_o = nc.variables['latitude'][:] lon_o = nc.variables['longitude'][:] @@ -106,7 +108,7 @@ class IoNetcdf(IoServer): path = os.path.join(netcdf_dir, '{0}_{1}{2}.nc'.format(var_name, date_array[0].year, str(date_array[0].month).zfill(2))) # self.logger.write_log('Getting temperature from {0}'.format(path), message_level=2) - + check_files(path) nc = Dataset(path, mode='r') lat_o = nc.variables['latitude'][:] lon_o = nc.variables['longitude'][:] diff --git a/hermesv3_bu/io_server/io_raster.py b/hermesv3_bu/io_server/io_raster.py index 030c8b82c6852ee4c8a85f5e81581b002d999d2d..9a9d73bfccaa18e2f0f8210c47028e409b832652 100755 --- a/hermesv3_bu/io_server/io_raster.py +++ b/hermesv3_bu/io_server/io_raster.py @@ -1,19 +1,16 @@ #!/usr/bin/env python -import sys import os import timeit -from warnings import warn from mpi4py import MPI import rasterio from rasterio.mask import mask import geopandas as gpd -import pandas as pd import numpy as np from shapely.geometry import Polygon - from hermesv3_bu.io_server.io_server import IoServer +from hermesv3_bu.tools.checker import check_files class IoRaster(IoServer): @@ -49,7 +46,7 @@ class IoRaster(IoServer): Function to parse features from GeoDataFrame in such a manner that rasterio wants them""" import json return [json.loads(gdf.to_json())['features'][0]['geometry']] - + check_files([raster_path, shape_path]) data = rasterio.open(raster_path) geo = gpd.read_file(shape_path) if len(geo) > 1: @@ -106,7 +103,7 @@ class IoRaster(IoServer): Function to parse features from GeoDataFrame in such a manner that rasterio wants them""" import json return [json.loads(gdf.to_json())['features'][0]['geometry']] - + check_files(raster_path) data = rasterio.open(raster_path) if len(geo) > 1: diff --git a/hermesv3_bu/io_server/io_shapefile.py b/hermesv3_bu/io_server/io_shapefile.py index f2c89d6b42010f53a186f07e011530e2d7483c3a..d4a5df150e46b3df3f6633e698cd8d5748745e48 100755 --- a/hermesv3_bu/io_server/io_shapefile.py +++ b/hermesv3_bu/io_server/io_shapefile.py @@ -10,6 +10,7 @@ import geopandas as gpd from mpi4py import MPI from hermesv3_bu.io_server.io_server import IoServer +from hermesv3_bu.tools.checker import check_files class IoShapefile(IoServer): @@ -59,13 +60,14 @@ class IoShapefile(IoServer): return True def read_shapefile_serial(self, path): - + check_files(path) gdf = gpd.read_file(path) return gdf def read_shapefile(self, path, rank=0): if self.comm.Get_rank() == rank: + check_files(path) gdf = gpd.read_file(path) gdf = np.array_split(gdf, self.comm.Get_size()) else: diff --git a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py index b291236e1daf9c768dfb7985acbf20a7cc3b527c..27bb100c6aeea020de8c5fd4e817077c2695dd2a 100755 --- a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py @@ -10,8 +10,9 @@ from hermesv3_bu.io_server.io_raster import IoRaster from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_netcdf import IoNetcdf from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files -formula = True +FORMULA = True class AgriculturalCropFertilizersSector(AgriculturalSector): @@ -23,6 +24,11 @@ class AgriculturalCropFertilizersSector(AgriculturalSector): crop_growing_degree_day_path): spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL CROP FERTILIZERS SECTOR =====') + check_files( + [nut_shapefile, land_uses_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, + molecular_weights_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, cultivated_ratio, + fertilizer_rate, crop_f_parameter, crop_f_fertilizers, gridded_ph, gridded_cec, crop_calendar, + temperature_path, wind_speed_path]) super(AgriculturalCropFertilizersSector, self).__init__( comm_agr, comm, logger, auxiliary_dir, grid, clip, date_array, nut_shapefile, source_pollutants, vertical_levels, crop_list, land_uses_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, None, None, @@ -93,7 +99,7 @@ class AgriculturalCropFertilizersSector(AgriculturalSector): for crop in self.crop_list: crop_ef = self.gridded_constants.loc[:, ['geometry', 'nut_code']].copy() # f_ph - if formula: + if FORMULA: # After Zhang et al. (2018) crop_ef['f_ph'] = (0.067 * self.gridded_constants['ph'] ** 2) - \ (0.69 * self.gridded_constants['ph']) + 0.68 diff --git a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py index f4566875672a834a830693871917f28ae168eb2d..9edb2a9cd4689096617ae453d2b3320dc92ab32a 100755 --- a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py @@ -8,6 +8,8 @@ import numpy as np from hermesv3_bu.sectors.agricultural_sector import AgriculturalSector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.logger.log import Log +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files class AgriculturalCropOperationsSector(AgriculturalSector): @@ -21,8 +23,8 @@ class AgriculturalCropOperationsSector(AgriculturalSector): not created yet. :type auxiliary_dir: str - :param grid_shp: Shapefile that contains the destination grid. It must contains the 'FID' (cell num). - :type grid_shp: GeoPandas.GeoDataframe + :param grid: Grid object. + :type grid: Grid :param clip: Path to the shapefile that contains the region of interest. :type clip: str @@ -89,6 +91,13 @@ class AgriculturalCropOperationsSector(AgriculturalSector): """ spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL CROP OPERATIONS SECTOR =====') + + check_files( + [nut_shapefile_path, land_uses_path, monthly_profiles_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path, + landuse_by_nut, crop_by_nut, crop_from_landuse_path] + + [os.path.join(ef_dir, ef_file) for ef_file in ['{0}.csv'.format(pol) for pol in source_pollutants]]) + super(AgriculturalCropOperationsSector, self).__init__( comm_agr, comm, logger, auxiliary_dir, grid, clip, date_array, nut_shapefile_path, source_pollutants, vertical_levels, crop_list, land_uses_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, ef_dir, diff --git a/hermesv3_bu/sectors/agricultural_machinery_sector.py b/hermesv3_bu/sectors/agricultural_machinery_sector.py index 7fb03bf4097c03b2d95d36caa7c5ce3976ffc8bd..25f965932c240a36c9797a782e6c6636756f157e 100755 --- a/hermesv3_bu/sectors/agricultural_machinery_sector.py +++ b/hermesv3_bu/sectors/agricultural_machinery_sector.py @@ -11,7 +11,7 @@ import numpy as np from hermesv3_bu.sectors.agricultural_sector import AgriculturalSector from hermesv3_bu.io_server.io_shapefile import IoShapefile -from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files class AgriculturalMachinerySector(AgriculturalSector): @@ -19,12 +19,18 @@ class AgriculturalMachinerySector(AgriculturalSector): vertical_levels, crop_list, nut_shapefile, machinery_list, land_uses_path, ef_files_dir, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, - machinery_distibution_nut_shapefile_path, deterioration_factor_path, load_factor_path, + machinery_distribution_nut_shapefile_path, deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, vehicle_workhours_path, vehicle_power_path, crop_machinery_nuts3): spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL MACHINERY SECTOR =====') + check_files( + [nut_shapefile, land_uses_path, ef_files_dir, monthly_profiles_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path, + landuse_by_nut, crop_by_nut, crop_from_landuse_path, machinery_distribution_nut_shapefile_path, + deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, + vehicle_workhours_path, vehicle_power_path, crop_machinery_nuts3]) super(AgriculturalMachinerySector, self).__init__( comm_agr, comm, logger, auxiliary_dir, grid, clip, date_array, nut_shapefile, source_pollutants, vertical_levels, crop_list, land_uses_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, @@ -35,7 +41,7 @@ class AgriculturalMachinerySector(AgriculturalSector): self.crop_machinery_nuts3 = self.read_profiles(crop_machinery_nuts3) self.crop_distribution = self.get_crop_distribution_by_nut( - self.crop_distribution, machinery_distibution_nut_shapefile_path, nut_code='nuts3_id') + self.crop_distribution, machinery_distribution_nut_shapefile_path, nut_code='nuts3_id') self.months = self.get_date_array_by_month() diff --git a/hermesv3_bu/sectors/aviation_sector.py b/hermesv3_bu/sectors/aviation_sector.py index 399efd3e8804fce6325a8850baa847d5300ad087..a18631b3d6f5017744fc82b4f8deb6dbde31c3d6 100755 --- a/hermesv3_bu/sectors/aviation_sector.py +++ b/hermesv3_bu/sectors/aviation_sector.py @@ -10,6 +10,8 @@ import geopandas as gpd from warnings import warn from hermesv3_bu.sectors.sector import Sector +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files, error_exit PHASE_TYPE = {'taxi_out': 'departure', 'pre-taxi_out': 'departure', 'takeoff': 'departure', 'climbout': 'departure', 'approach': 'arrival', 'taxi_in': 'arrival', 'post-taxi_in': 'arrival', 'landing': 'arrival', @@ -50,8 +52,8 @@ class AviationSector(Sector): created yet. :type auxiliary_dir: str - :param grid_shp: Shapefile with the grid horizontal distribution. - :type grid_shp: GeoDataFrame + :param grid: Grid object. + :type grid: Grid :param date_array: List of datetimes. :type date_array: list(datetime.datetime, ...) @@ -123,8 +125,16 @@ class AviationSector(Sector): file must contain the 'Specie' and 'MW' columns. :type molecular_weights_path: str """ + spent_time = timeit.default_timer() + logger.write_log('===== AVIATION SECTOR =====') + check_files( + [airport_shapefile_path, airport_runways_shapefile_path, airport_runways_corners_shapefile_path, + airport_trajectories_shapefile_path, operations_path, planes_path, times_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path] + + [os.path.join(ef_dir, PHASE_EF_FILE[phase]) for phase in PHASE_TYPE.keys()]) + super(AviationSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, None, weekly_profiles_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, @@ -295,7 +305,7 @@ class AviationSector(Sector): operations = operations.loc[operations['plane_id'].isin(self.plane_list), :] if len(operations) == 0: - raise NameError("The plane/s defined in the plane_list do not exist.") + error_exit("The plane/s defined in the plane_list do not exist.") operations = operations.loc[operations['airport_id'].isin(self.airport_list), :] operations.set_index(['airport_id', 'plane_id', 'operation'], inplace=True) operations.rename(columns={'1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, '10': 10, @@ -373,8 +383,8 @@ class AviationSector(Sector): shp_airport_list = list(set(conf_airport_list).intersection(shp_airport_list)) if len(shp_airport_list) == 0: - raise NameError("No airports intersect with the defined domain or the defined aiport/s in the " + - "airport_list do no exist ") + error_exit("No airports intersect with the defined domain or the defined aiport/s in the " + + "airport_list do no exist ") airports_with_operations = np.unique(pd.read_csv(operations_file, usecols=['airport_id']).values) @@ -391,9 +401,9 @@ class AviationSector(Sector): for i in range(self.comm.size)] for sublist in new_list: if len(sublist) == 0: - raise ValueError("ERROR: The selected number of processors is to high. " + - "The maximum number of processors accepted are {0}".format(max_len) + - "(Maximum number of airports included in the working domain") + error_exit("The selected number of processors is to high. " + + "The maximum number of processors accepted are {0}".format(max_len) + + "(Maximum number of airports included in the working domain") else: new_list = None diff --git a/hermesv3_bu/sectors/livestock_sector.py b/hermesv3_bu/sectors/livestock_sector.py index bede31112c7220c875cc4885024c98abfb0c34dd..2cf1cc921d2a760747e063139ec5d166d3f4fa2d 100755 --- a/hermesv3_bu/sectors/livestock_sector.py +++ b/hermesv3_bu/sectors/livestock_sector.py @@ -12,6 +12,8 @@ from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_raster import IoRaster from hermesv3_bu.io_server.io_netcdf import IoNetcdf +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files, error_exit # Constants for grassing daily factor estimation SIGMA = 60 @@ -43,8 +45,8 @@ class LivestockSector(Sector): ['nox_no', 'nh3', 'nmvoc', 'pm10', 'pm25'] :type source_pollutants: list - :param grid_shp: Shapefile that contains the destination grid. It must contains the 'FID' (cell num). - :type grid_shp: GeoPandas.GeoDataframe + :param grid: Grid object. + :type grid: Grid :param clip: Clip. :type clip: Clip @@ -153,6 +155,18 @@ class LivestockSector(Sector): """ spent_time = timeit.default_timer() logger.write_log('===== LIVESTOCK SECTOR =====') + check_files( + [gridded_livestock_path.replace('', animal) for animal in animal_list] + + [correction_split_factors_path.replace('', animal) for animal in animal_list] + + [temperature_dir, wind_speed_dir, + denominator_yearly_factor_dir, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, + speciation_map_path, speciation_profiles_path, molecular_weights_path, nut_shapefile_path] + + [os.path.join(ef_dir, ef_file) for ef_file in + ['{0}.csv'.format(pol) for pol in source_pollutants if pol not in ['pm10', 'pm25']]]) + for pol in source_pollutants: + if pol in ['pm10', 'pm25']: + check_files(os.path.join(ef_dir, 'pm.csv')) + super(LivestockSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -722,7 +736,7 @@ class LivestockSector(Sector): (animals_df[animal['Code']] * animals_df['FD_housing_closed']).multiply( animal['EF_housing']) else: - raise KeyError('Animal {0} not found on the nh3 emission factors file.'.format(animal.Code)) + error_exit('Animal {0} not found on the nh3 emission factors file.'.format(animal.Code)) # Storage emissions out_df.loc[:, out_p] += \ (animals_df[animal['Code']] * animals_df['FD_storage']).multiply(animal['EF_yarding']) diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index b9555a90844c3d9b5cb101c2de9e8bc1a12f099a..872470c0153f5b9749c47cf5cea4c7927698c489 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -10,6 +10,7 @@ from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile # from hermesv3_bu.io_server.io_netcdf import IoNetcdf from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files, error_exit INTERPOLATION_TYPE = 'linear' # GRAVITI m/s-2 @@ -22,8 +23,8 @@ class PointSourceSector(Sector): """ Class to calculate the Point Source emissions - :param grid_shp: Grid of the destination domain - :type grid_shp: Grid + :param grid: Grid of the destination domain + :type grid: Grid :param catalog_path: Path to the fine that contains all the information for each point source. :type catalog_path: str @@ -51,7 +52,10 @@ class PointSourceSector(Sector): speciation_map_path, speciation_profiles_path, sector_list, measured_emission_path, molecular_weights_path, plume_rise=False, plume_rise_pahts=None): spent_time = timeit.default_timer() - + logger.write_log('===== POINT SOURCES SECTOR =====') + check_files( + [catalog_path, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, + speciation_profiles_path]) super(PointSourceSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -59,13 +63,43 @@ class PointSourceSector(Sector): self.plume_rise = plume_rise self.catalog = self.read_catalog_shapefile(catalog_path, sector_list) - + self.check_catalog() self.catalog_measured = self.read_catalog_for_measured_emissions(catalog_path, sector_list) self.measured_path = measured_emission_path self.plume_rise_pahts = plume_rise_pahts self.logger.write_time_log('PointSourceSector', '__init__', timeit.default_timer() - spent_time) + def check_catalog(self): + # Checking monthly profiles IDs + links_month = set(np.unique(self.catalog['P_month'].dropna().values)) + month = set(self.monthly_profiles.index.values) + month_res = links_month - month + if len(month_res) > 0: + error_exit("The following monthly profile IDs reported in the point sources shapefile do not appear " + + "in the monthly profiles file. {0}".format(month_res)) + # Checking weekly profiles IDs + links_week = set(np.unique(self.catalog['P_week'].dropna().values)) + week = set(self.weekly_profiles.index.values) + week_res = links_week - week + if len(week_res) > 0: + error_exit("The following weekly profile IDs reported in the point sources shapefile do not appear " + + "in the weekly profiles file. {0}".format(week_res)) + # Checking hourly profiles IDs + links_hour = set(np.unique(self.catalog['P_hour'].dropna().values)) + hour = set(self.hourly_profiles.index.values) + hour_res = links_hour - hour + if len(hour_res) > 0: + error_exit("The following hourly profile IDs reported in the point sources shapefile do not appear " + + "in the hourly profiles file. {0}".format(hour_res)) + # Checking specly profiles IDs + links_spec = set(np.unique(self.catalog['P_spec'].dropna().values)) + spec = set(self.speciation_profile.index.values) + spec_res = links_spec - spec + if len(spec_res) > 0: + error_exit("The following speciation profile IDs reported in the point sources shapefile do not appear " + + "in the speciation profiles file. {0}".format(month_res)) + def read_catalog_csv(self, catalog_path, sector_list): """ Read the catalog @@ -782,7 +816,7 @@ class PointSourceSector(Sector): try: test.set_index(x.index, inplace=True) except ValueError: - raise IOError('No measured emissions for the selected dates: {0}'.format(x.values)) + error_exit('No measured emissions for the selected dates: {0}'.format(x.values)) return test[pollutant] diff --git a/hermesv3_bu/sectors/recreational_boats_sector.py b/hermesv3_bu/sectors/recreational_boats_sector.py index 462c7d65247bfc2315e248224039615dd5eb5f2a..4b7c5c7cce7ecf40c9b78cdfc762d7e9583f0b81 100755 --- a/hermesv3_bu/sectors/recreational_boats_sector.py +++ b/hermesv3_bu/sectors/recreational_boats_sector.py @@ -10,7 +10,7 @@ import geopandas as gpd from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_raster import IoRaster -from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files class RecreationalBoatsSector(Sector): @@ -19,7 +19,10 @@ class RecreationalBoatsSector(Sector): weekly_profiles_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path): spent_time = timeit.default_timer() - + logger.write_log('===== RECREATIONAL BOATS SECTOR =====') + check_files( + [density_map_path, boats_data_path, ef_file_path, monthly_profiles_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path]) super(RecreationalBoatsSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, diff --git a/hermesv3_bu/sectors/residential_sector.py b/hermesv3_bu/sectors/residential_sector.py index 17f6904e98f510caaf457ded6cbd32a1ef1d04d8..d098775f7e9346b0f6cb2ed9f27ecca03a0cd7e3 100755 --- a/hermesv3_bu/sectors/residential_sector.py +++ b/hermesv3_bu/sectors/residential_sector.py @@ -12,7 +12,7 @@ from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_raster import IoRaster from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_netcdf import IoNetcdf -from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files class ResidentialSector(Sector): @@ -23,7 +23,12 @@ class ResidentialSector(Sector): heating_degree_day_path, temperature_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path): spent_time = timeit.default_timer() - + logger.write_log('===== RESIDENTIAL COMBUSTION SECTOR =====') + check_files( + [prov_shapefile, ccaa_shapefile, population_density_map, population_type_map, population_type_nuts2, + population_type_nuts3, energy_consumption_nuts3, energy_consumption_nuts2, residential_spatial_proxies, + residential_ef_files_path, temperature_path, hourly_profiles_path, speciation_map_path, + speciation_profiles_path, molecular_weights_path]) super(ResidentialSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, None, None, hourly_profiles_path, speciation_map_path, diff --git a/hermesv3_bu/sectors/shipping_port_sector.py b/hermesv3_bu/sectors/shipping_port_sector.py index 0e316c2f701022bb828af4e6440ac586da41d73e..88759a2bc2ae2cde8aa395f63acb6ab6d5a0c464 100755 --- a/hermesv3_bu/sectors/shipping_port_sector.py +++ b/hermesv3_bu/sectors/shipping_port_sector.py @@ -7,6 +7,8 @@ import numpy as np import timeit from hermesv3_bu.logger.log import Log from hermesv3_bu.io_server.io_shapefile import IoShapefile +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files, error_exit class ShippingPortSector(Sector): @@ -28,8 +30,8 @@ class ShippingPortSector(Sector): created yet. :type auxiliary_dir: str - :param grid_shp: Shapefile with the grid horizontal distribution. - :type grid_shp: GeoDataFrame + :param grid: Grid object. + :type grid: Grid :param date_array: List of datetimes. :type date_array: list(datetime.datetime, ...) @@ -82,6 +84,11 @@ class ShippingPortSector(Sector): spent_time = timeit.default_timer() logger.write_log('===== SHIPPING PORT SECTOR =====') + check_files( + [hoteling_shapefile_path, maneuvering_shapefile_path, engine_percent_path, tonnage_path, load_factor_path, + power_path, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, + speciation_profiles_path, molecular_weights_path, ef_dir]) + super(ShippingPortSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -111,7 +118,7 @@ class ShippingPortSector(Sector): port_shp = gpd.sjoin(port_shp, self.clip.shapefile.to_crs(port_shp.crs), how='inner', op='intersects') port_list = np.unique(port_shp['code'].values) if len(port_list) < self.comm.Get_size(): - raise ValueError("The chosen number of processors {0} exceeds the number of involved ports {1}.".format( + error_exit("The chosen number of processors {0} exceeds the number of involved ports {1}.".format( self.comm.Get_size(), len(port_list)) + " Set {0} at shipping_port_processors value.".format( len(port_list))) port_list = np.array_split(port_list, self.comm.Get_size()) @@ -120,8 +127,6 @@ class ShippingPortSector(Sector): port_list = self.comm.scatter(port_list, root=0) - if len(port_list) == 0: - raise ValueError("The number ") return list(port_list) def read_monthly_profiles(self, path): diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 30479546d91bf2841253d33f84ef71eaa2b0d6bc..21045250b39845fbd10f5d1cf9bb6d67f2560333 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -9,6 +9,8 @@ import numpy as np from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_netcdf import IoNetcdf +from hermesv3_bu.tools.checker import check_files, error_exit + pmc_list = ['pmc', 'PMC'] @@ -22,7 +24,14 @@ class TrafficAreaSector(Sector): small_cities_monthly_profile, small_cities_weekly_profile, small_cities_hourly_profile): spent_time = timeit.default_timer() logger.write_log('===== TRAFFIC AREA SECTOR =====') - + if do_evaporative: + check_files([population_tif_path, speciation_map_path, molecular_weights_path, + gasoline_path, total_pop_by_prov, nuts_shapefile, speciation_profiles_evaporative, + evaporative_ef_file, temperature_dir]) + if do_small_cities: + check_files([population_tif_path, speciation_map_path, molecular_weights_path, + small_cities_shp, speciation_profiles_small_cities, small_cities_ef_file, + small_cities_monthly_profile, small_cities_weekly_profile, small_cities_hourly_profile]) super(TrafficAreaSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, None, None, None, speciation_map_path, None, molecular_weights_path) @@ -456,7 +465,7 @@ class TrafficAreaSector(Sector): elif self.do_small_cities: dataset = self.small_cities else: - raise ValueError('No traffic area emission selected. do_evaporative and do_small_cities are False') + error_exit('No traffic area emission selected. do_evaporative and do_small_cities are False') dataset['layer'] = 0 dataset = dataset.groupby(['FID', 'layer', 'tstep']).sum() diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 41fcb2fb85ed9a0f4e50f9d6d10f51087185acf1..ee02526e08905674125ebcf3de3336300b1b62e1 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -13,6 +13,7 @@ import warnings from hermesv3_bu.logger.log import Log from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_netcdf import IoNetcdf +from hermesv3_bu.tools.checker import check_files, error_exit from ctypes import cdll, CDLL cdll.LoadLibrary("libc.so.6") @@ -55,6 +56,45 @@ class TrafficSector(Sector): spent_time = timeit.default_timer() logger.write_log('===== TRAFFIC SECTOR =====') + if do_hot: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, hot_cold_speciation] + + [os.path.join(ef_common_path, "hot_{0}.csv".format(pol)) for pol in source_pollutants]) + if do_cold: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, hot_cold_speciation, + temp_common_path] + + [os.path.join(ef_common_path, "cold_{0}.csv".format(pol)) for pol in source_pollutants]) + if do_tyre_wear: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, tyre_speciation] + + [os.path.join(ef_common_path, "tyre_{0}.csv".format(pol)) for pol in ['pm']]) + if do_road_wear: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, road_speciation] + + [os.path.join(ef_common_path, "road_{0}.csv".format(pol)) for pol in ['pm']]) + if do_brake_wear: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, brake_speciation] + + [os.path.join(ef_common_path, "brake_{0}.csv".format(pol)) for pol in ['pm']]) + if do_resuspension: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, resuspension_speciation] + + [os.path.join(ef_common_path, "resuspension_{0}.csv".format(pol)) for pol in ['pm']]) + if resuspension_correction: + check_files(precipitation_path) super(TrafficSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, None, speciation_map_path, None, molecular_weights_path) @@ -121,24 +161,24 @@ class TrafficSector(Sector): speed_res = links_speed - speed if len(speed_res) > 0: - raise ValueError("The following speed profile IDs reported in the road links shapefile do not appear " + - "in the hourly speed profiles file. {0}".format(speed_res)) + error_exit("The following speed profile IDs reported in the road links shapefile do not appear " + + "in the hourly speed profiles file. {0}".format(speed_res)) # Checking monthly profiles IDs links_month = set(np.unique(self.road_links['aadt_m_mn'].dropna().values)) month = set(self.monthly_profiles.index.values) month_res = links_month - month if len(month_res) > 0: - raise ValueError("The following monthly profile IDs reported in the road links shapefile do not appear " + - "in the monthly profiles file. {0}".format(month_res)) + error_exit("The following monthly profile IDs reported in the road links shapefile do not appear " + + "in the monthly profiles file. {0}".format(month_res)) # Checking weekly profiles IDs links_week = set(np.unique(self.road_links['aadt_week'].dropna().values)) week = set(self.weekly_profiles.index.values) week_res = links_week - week if len(week_res) > 0: - raise ValueError("The following weekly profile IDs reported in the road links shapefile do not appear " + - "in the weekly profiles file. {0}".format(week_res)) + error_exit("The following weekly profile IDs reported in the road links shapefile do not appear " + + "in the weekly profiles file. {0}".format(week_res)) # Checking hourly profiles IDs links_hour = set(np.unique(np.concatenate([ @@ -150,8 +190,8 @@ class TrafficSector(Sector): hour = set(self.hourly_profiles.index.values) hour_res = links_hour - hour if len(hour_res) > 0: - raise ValueError("The following hourly profile IDs reported in the road links shapefile do not appear " + - "in the hourly profiles file. {0}".format(hour_res)) + error_exit("The following hourly profile IDs reported in the road links shapefile do not appear " + + "in the hourly profiles file. {0}".format(hour_res)) self.logger.write_time_log('TrafficSector', 'check_profiles', timeit.default_timer() - spent_time) @@ -304,8 +344,11 @@ class TrafficSector(Sector): if self.comm.Get_rank() == 0: df = gpd.read_file(path) - df.drop(columns=['Adminis', 'CCAA', 'NETWORK_ID', 'Province', 'Road_name', 'aadt_m_sat', 'aadt_m_sun', - 'aadt_m_wd', 'Source'], inplace=True) + try: + df.drop(columns=['Adminis', 'CCAA', 'NETWORK_ID', 'Province', 'Road_name', 'aadt_m_sat', 'aadt_m_sun', + 'aadt_m_wd', 'Source'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) libc.malloc_trim(0) # df.to_file('~/temp/road_links.shp') df = gpd.sjoin(df, self.clip.shapefile.to_crs(df.crs), how="inner", op='intersects') @@ -318,7 +361,11 @@ class TrafficSector(Sector): df = df[df['CONS'] != 0] df = df[df['aadt'] > 0] - df.drop(columns=['CONS'], inplace=True) + try: + df.drop(columns=['CONS'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) + df = df.loc[df['aadt_m_mn'] != 'NULL', :] libc.malloc_trim(0) @@ -357,7 +404,7 @@ class TrafficSector(Sector): # Check if percents are ok if len(df[df['PcLight'] < 0]) is not 0: - raise ValueError('ERROR: PcLight < 0') + error_exit('PcLight < 0') if self.write_rline: self.write_rline_roadlinks(df) @@ -389,7 +436,10 @@ class TrafficSector(Sector): # Pollutants different to NH3 if pollutant_name != 'nh3': - del df['Copert_V_name'] + try: + df.drop(columns=['Copert_V_name'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) # For hot emission factors if emission_type == 'hot': @@ -397,8 +447,10 @@ class TrafficSector(Sector): df.loc[df['Technology'].isnull(), 'Technology'] = '' df = df[df['Technology'] != 'EGR'] - - del df['Technology'], df['Load'] + try: + df.drop(columns=['Technology', 'Load'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) # Split the EF file into small DataFrames divided by column Road.Slope and Mode restrictions. df_code_slope_road = df[df['Road.Slope'].notnull() & df['Mode'].notnull()] @@ -409,7 +461,7 @@ class TrafficSector(Sector): # Checks that the splited DataFrames contain the full DataFrame if (len(df_code_slope_road) + len(df_code_slope) + len(df_code_road) + len(df_code)) != len(df): # TODO check that error - raise ValueError('ERROR in blablavbla') + error_exit('ERROR in blablavbla') return df_code_slope_road, df_code_slope, df_code_road, df_code elif emission_type == 'cold' or emission_type == 'tyre' or emission_type == 'road' or \ @@ -417,7 +469,10 @@ class TrafficSector(Sector): return df # NH3 pollutant else: - del df['Copert_V_name'] + try: + df.drop(columns=['Copert_V_name'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) # Specific case for cold NH3 emission factors that needs the hot emission factors and the cold ones. if emission_type == 'cold': df_hot = self.read_ef('hot', pollutant_name) @@ -425,8 +480,10 @@ class TrafficSector(Sector): df = df.merge(df_hot, left_on=['Code', 'Mode'], right_on=['Code_hot', 'Mode_hot'], how='left') - - del df['Cmileage_hot'], df['Mode_hot'], df['Code_hot'] + try: + df.drop(columns=['Cmileage_hot', 'Mode_hot', 'Code_hot'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) return df @@ -512,7 +569,10 @@ class TrafficSector(Sector): df = df[df['Fleet_value'] > 0] # Deleting unused columns - df.drop(columns=['aadt', 'PcLight', 'PcHeavy', 'PcMoto', 'PcMoped', 'Fleet_Class'], inplace=True) + try: + df.drop(columns=['aadt', 'PcLight', 'PcHeavy', 'PcMoto', 'PcMoped', 'Fleet_Class'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) libc.malloc_trim(0) self.logger.write_time_log('TrafficSector', 'update_fleet_value', timeit.default_timer() - spent_time) @@ -569,7 +629,7 @@ class TrafficSector(Sector): elif weekday == 6: return 'aadt_h_sun' else: - raise KeyError('ERROR: Weekday not found') + error_exit('Weekday not found') # Monthly factor x = pd.merge(x, self.monthly_profiles, left_on='aadt_m_mn', right_index=True, how='left') @@ -606,10 +666,13 @@ class TrafficSector(Sector): df[['month', 'weekday', 'hour', 'aadt_m_mn', 'aadt_week', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun']]) - df.drop(columns=['month', 'weekday', 'hour', 'P_speed', 'speed_mean', 'sp_wd', 'sp_we', 'sp_hour_mo', - 'sp_hour_tu', 'sp_hour_we', 'sp_hour_th', 'sp_hour_fr', 'sp_hour_sa', 'sp_hour_su', 'aux_date', - 'aadt_m_mn', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun', 'aadt_week', 'start_date'], - inplace=True) + try: + df.drop(columns=['month', 'weekday', 'hour', 'P_speed', 'speed_mean', 'sp_wd', 'sp_we', 'sp_hour_mo', + 'sp_hour_tu', 'sp_hour_we', 'sp_hour_th', 'sp_hour_fr', 'sp_hour_sa', 'sp_hour_su', + 'aux_date', 'aadt_m_mn', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun', 'aadt_week', + 'start_date'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) libc.malloc_trim(0) self.logger.write_time_log('TrafficSector', 'calculate_time_dependent_values', @@ -650,7 +713,7 @@ class TrafficSector(Sector): try: fleet = self.fleet_compo[['Code', 'Class', zone]] except KeyError as e: - raise KeyError(e.message + ' of the fleet_compo file') + error_exit(e.message + ' of the fleet_compo file') fleet.columns = ['Fleet_Code', 'Fleet_Class', 'Fleet_value'] fleet = fleet[fleet['Fleet_value'] > 0] @@ -690,7 +753,7 @@ class TrafficSector(Sector): expanded_aux = expanded_aux.merge(ef_code_road, left_on=['Fleet_Code', 'Road_type'], right_on=['Code', 'Mode'], how='inner') - del expanded_aux['Code'], expanded_aux['Mode'] + expanded_aux.drop(columns=['Code', 'Mode'], inplace=True) # Warnings and Errors original_ef_profile = np.unique(self.expanded.index.get_level_values('Fleet_Code')) @@ -703,7 +766,7 @@ class TrafficSector(Sector): resta_1)) warnings.warn('Exists some fleet codes that not appear on the EF file: {0}'.format(resta_1), Warning) if len(resta_2) > 0: - raise ImportError('Exists some fleet codes duplicateds on the EF file: {0}'.format(resta_2)) + error_exit('Exists some fleet codes duplicated on the EF file: {0}'.format(resta_2)) m_corr = self.read_mcorr_file(pollutant) if m_corr is not None: @@ -777,13 +840,10 @@ class TrafficSector(Sector): spent_time = timeit.default_timer() cold_links = self.road_links.copy().reset_index() - - del cold_links['aadt'], cold_links['PcHeavy'], cold_links['PcMoto'], cold_links['PcMoped'], cold_links['sp_wd'] - del cold_links['sp_we'], cold_links['sp_hour_su'], cold_links['sp_hour_mo'], cold_links['sp_hour_tu'] - del cold_links['sp_hour_we'], cold_links['sp_hour_th'], cold_links['sp_hour_fr'], cold_links['sp_hour_sa'] - del cold_links['Road_type'], cold_links['aadt_m_mn'], cold_links['aadt_h_mn'], cold_links['aadt_h_wd'] - del cold_links['aadt_h_sat'], cold_links['aadt_h_sun'], cold_links['aadt_week'], cold_links['fleet_comp'] - del cold_links['road_grad'], cold_links['PcLight'], cold_links['start_date'] + cold_links.drop(columns=['aadt', 'PcHeavy', 'PcMoto', 'PcMoped', 'sp_wd', 'sp_we', 'sp_hour_su', 'sp_hour_mo', + 'sp_hour_tu', 'sp_hour_we', 'sp_hour_th', 'sp_hour_fr', 'sp_hour_sa', 'Road_type', + 'aadt_m_mn', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun', 'aadt_week', + 'fleet_comp', 'road_grad', 'PcLight', 'start_date'], inplace=True) libc.malloc_trim(0) cold_links['centroid'] = cold_links['geometry'].centroid @@ -801,11 +861,12 @@ class TrafficSector(Sector): unary_union = temperature.unary_union cold_links['REC'] = cold_links.apply(self.nearest, geom_union=unary_union, df1=cold_links, df2=temperature, geom1_col='centroid', src_column='REC', axis=1) - del cold_links['geometry'], cold_links['centroid'], temperature['geometry'] + + cold_links.drop(columns=['geometry', 'centroid', 'geometry'], inplace=True) libc.malloc_trim(0) cold_links = cold_links.merge(temperature, left_on='REC', right_on='REC', how='left') - del cold_links['REC'] + cold_links.drop(columns=['REC'], inplace=True) libc.malloc_trim(0) c_expanded = hot_expanded.merge(cold_links, left_on='Link_ID', right_on='Link_ID', how='left') @@ -874,18 +935,18 @@ class TrafficSector(Sector): uni.remove(o) except Exception: error_fleet_code.append(o) - raise IndexError('There are duplicated values for {0} codes in the cold EF files.'.format(error_fleet_code)) + error_exit('There are duplicated values for {0} codes in the cold EF files.'.format(error_fleet_code)) for tstep in range(len(self.date_array)): if 'pm' in self.source_pollutants: cold_df['pm10_{0}'.format(tstep)] = cold_df['pm_{0}'.format(tstep)] cold_df['pm25_{0}'.format(tstep)] = cold_df['pm_{0}'.format(tstep)] - del cold_df['pm_{0}'.format(tstep)] + cold_df.drop(columns=['pm_{0}'.format(tstep)], inplace=True) libc.malloc_trim(0) if 'voc' in self.source_pollutants and 'ch4' in self.source_pollutants: cold_df['nmvoc_{0}'.format(tstep)] = \ cold_df['voc_{0}'.format(tstep)] - cold_df['ch4_{0}'.format(tstep)] - del cold_df['voc_{0}'.format(tstep)] + cold_df.drop(columns=['voc_{0}'.format(tstep)], inplace=True) libc.malloc_trim(0) else: self.logger.write_log("WARNING! nmvoc emissions cannot be estimated because voc or ch4 are not " + @@ -902,21 +963,20 @@ class TrafficSector(Sector): spent_time = timeit.default_timer() columns_to_delete = ['Road_type', 'Fleet_value'] + ['v_{0}'.format(x) for x in range(len(self.date_array))] - for column_name in columns_to_delete: - del expanded[column_name] + expanded.drop(columns=columns_to_delete, inplace=True) for tstep in range(len(self.date_array)): if 'pm' in self.source_pollutants: expanded['pm10_{0}'.format(tstep)] = expanded['pm_{0}'.format(tstep)] expanded['pm25_{0}'.format(tstep)] = expanded['pm_{0}'.format(tstep)] - del expanded['pm_{0}'.format(tstep)] + expanded.drop(columns=['pm_{0}'.format(tstep)], inplace=True) if 'voc' in self.source_pollutants and 'ch4' in self.source_pollutants: expanded['nmvoc_{0}'.format(tstep)] = expanded['voc_{0}'.format(tstep)] - \ expanded['ch4_{0}'.format(tstep)] # For certain vehicles (mostly diesel) and speeds, in urban road CH4 > than VOC according to COPERT V expanded.loc[expanded['nmvoc_{0}'.format(tstep)] < 0, 'nmvoc_{0}'.format(tstep)] = 0 - del expanded['voc_{0}'.format(tstep)] + expanded.drop(columns=['voc_{0}'.format(tstep)], inplace=True) else: self.logger.write_log("nmvoc emissions cannot be estimated because voc or ch4 are not selected in " + "the pollutant list.") @@ -950,7 +1010,7 @@ class TrafficSector(Sector): if pollutant == 'pm': df['pm10_{0}'.format(tstep)] = df[p_column] * 0.6 df['pm25_{0}'.format(tstep)] = df[p_column] * 0.42 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -983,7 +1043,7 @@ class TrafficSector(Sector): if pollutant == 'pm': df.loc[:, 'pm10_{0}'.format(tstep)] = df[p_column] * 0.98 df.loc[:, 'pm25_{0}'.format(tstep)] = df[p_column] * 0.39 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -1014,7 +1074,7 @@ class TrafficSector(Sector): if pollutant == 'pm': df.loc[:, 'pm10_{0}'.format(tstep)] = df[p_column] * 0.5 df.loc[:, 'pm25_{0}'.format(tstep)] = df[p_column] * 0.27 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -1043,11 +1103,12 @@ class TrafficSector(Sector): unary_union = p_factor.unary_union road_link_aux['REC'] = road_link_aux.apply(self.nearest, geom_union=unary_union, df1=road_link_aux, df2=p_factor, geom1_col='centroid', src_column='REC', axis=1) - del road_link_aux['centroid'], p_factor['geometry'] + road_link_aux.drop(columns=['centroid'], inplace=True) + p_factor.drop(columns=['geometry'], inplace=True) road_link_aux = road_link_aux.merge(p_factor, left_on='REC', right_on='REC', how='left') - del road_link_aux['REC'] + road_link_aux.drop(columns=['REC'], inplace=True) pollutants = ['pm'] for pollutant in pollutants: @@ -1056,7 +1117,7 @@ class TrafficSector(Sector): if self.resuspension_correction: df = df.merge(road_link_aux, left_on='Link_ID', right_on='Link_ID', how='left') - del df['road_grad'], df['Road_type'], df['Code'] + df.drop(columns=['road_grad', 'Road_type', 'Code'], inplace=True) for tstep in range(len(self.date_array)): p_column = '{0}_{1}'.format(pollutant, tstep) f_column = 'f_{0}'.format(tstep) @@ -1071,7 +1132,7 @@ class TrafficSector(Sector): df['pm10_{0}'.format(tstep)] = df[p_column] # TODO Check fraction of pm2.5 df['pm25_{0}'.format(tstep)] = df[p_column] * 0.5 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -1112,7 +1173,7 @@ class TrafficSector(Sector): # Reads speciation profile speciation = self.read_profiles(speciation) - del speciation['Copert_V_name'] + speciation.drop(columns=['Copert_V_name'], inplace=True) # Transform dataset into timestep rows instead of timestep columns df = self.transform_df(df) @@ -1164,7 +1225,7 @@ class TrafficSector(Sector): else: mol_w = self.molecular_weights[in_p] except KeyError: - raise AttributeError('{0} not found in the molecular weights file.'.format(in_p)) + error_exit('{0} not found in the molecular weights file.'.format(in_p)) # from g/km.h to mol/km.h or g/km.h (aerosols) df_aux[p] = df_aux.loc[:, p] / mol_w @@ -1173,7 +1234,7 @@ class TrafficSector(Sector): df_out_list.append(df_aux[[p] + ['tstep', 'Link_ID']].groupby(['tstep', 'Link_ID']).sum()) del df_aux - del df[in_p] + df.drop(columns=[in_p], inplace=True) df_out = pd.concat(df_out_list, axis=1) @@ -1300,7 +1361,7 @@ class TrafficSector(Sector): link_grid = pd.read_csv(self.link_to_grid_csv) link_grid = link_grid[link_grid['Link_ID'].isin(link_emissions['Link_ID'].values)] - del link_emissions['geometry'] + link_emissions.drop(columns=['geometry'], inplace=True) link_grid = link_grid.merge(link_emissions, left_on='Link_ID', right_on='Link_ID') if 'Unnamed: 0' in link_grid.columns.values: link_grid.drop(columns=['Unnamed: 0'], inplace=True) @@ -1311,8 +1372,7 @@ class TrafficSector(Sector): cols_to_update.remove('FID') for col in cols_to_update: link_grid.loc[:, col] = link_grid[col] * link_grid['length'] - del link_grid['length'] - link_grid.drop(columns=['Link_ID'], inplace=True) + link_grid.drop(columns=['length', 'Link_ID'], inplace=True) link_grid['layer'] = 0 link_grid = link_grid.groupby(['FID', 'layer', 'tstep']).sum() diff --git a/hermesv3_bu/tools/checker.py b/hermesv3_bu/tools/checker.py new file mode 100644 index 0000000000000000000000000000000000000000..e13b06d219ff43c87e0a81e466773b606c61bcd4 --- /dev/null +++ b/hermesv3_bu/tools/checker.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +import os +import sys +from mpi4py import MPI +from warnings import warn + + +def check_files(file_path_list, warning=False): + if isinstance(file_path_list, str): + file_path_list = [file_path_list] + + files_not_found = [] + for file_path in file_path_list: + if not os.path.exists(file_path): + files_not_found.append(file_path) + + if len(files_not_found) > 0: + error_message = "*ERROR* (Rank {0}) File/s not found:".format(MPI.COMM_WORLD.Get_rank()) + for file_path in files_not_found: + error_message += "\n\t{0}".format(file_path) + + if warning: + warn(error_message.replace('ERROR', 'WARNING')) + return False + else: + error_exit(error_message) + return True + + +def error_exit(error_message): + if not error_message[:7] == "*ERROR*": + error_message = "*ERROR* (Rank {0}) ".format(MPI.COMM_WORLD.Get_rank()) + error_message + print(error_message, file=sys.stderr) + MPI.COMM_WORLD.Abort()