From 2c9afda95e685873a59bca745c384552c5efe5d6 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 10:30:27 +0200 Subject: [PATCH 01/16] Added checker --- hermesv3_bu/tools/checker.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 hermesv3_bu/tools/checker.py diff --git a/hermesv3_bu/tools/checker.py b/hermesv3_bu/tools/checker.py new file mode 100644 index 0000000..27c2c0a --- /dev/null +++ b/hermesv3_bu/tools/checker.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +import os +import sys +from mpi4py import MPI + + +def check_file(file_path_list, warning=False): + if isinstance(file_path_list, str): + file_path_list = [file_path_list] + + files_not_found = [] + for file_path in file_path_list: + if not os.path.exists(file_path): + files_not_found.append(file_path) + + if len(files_not_found) > 0: + error_message = "*ERROR* (Rank {0}) File/s not found:".format(MPI.COMM_WORLD.Get_rank()) + for file_path in files_not_found: + error_message += "\n\t{0}".format(file_path) + print(error_message, file=sys.stderr) + MPI.COMM_WORLD.Abort() -- GitLab From fb92c1749ad1bc8c372453092b90657d8d35d8e6 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 11:01:07 +0200 Subject: [PATCH 02/16] - Aviation: Checked files and raises --- conf/hermes.conf | 12 ++++++------ hermesv3_bu/sectors/aviation_sector.py | 26 ++++++++++++++++++-------- hermesv3_bu/tools/checker.py | 18 +++++++++++++++--- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 0858957..b613126 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -2,14 +2,14 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon -output_dir = /scratch/Earth/HERMESv3_BU_OUT/parallel -output_name = HERMESv3__residential.nc +output_dir = /scratch/Earth/HERMESv3_BU_OUT +output_name = HERMESv3__aviation.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 output_timestep_num = 24 -auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/parallel/_ +auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ erase_auxiliary_files = 0 @@ -130,13 +130,13 @@ writing_processors = 1 # traffic_processors = 256 # traffic_area_processors = 1 -aviation_processors = 0 +aviation_processors = 1 shipping_port_processors = 0 livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 -residential_processors = 2 +residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 traffic_processors = 0 @@ -175,7 +175,7 @@ layer_thickness_dir = /esarchive/exp/monarch/a1wd/regional/hourly/layer_thicknes aviation_source_pollutants = nox_no2, co, hc, so2, pm10, pm25, co2 # airport_list = # plane_list = -airport_shapefile_path = /aviation/Airports.shp +airport_shapefile_path = /aviation/Airports.shpX airport_runways_shapefile_path = /aviation/Runways.shp airport_runways_corners_shapefile_path = /aviation/Runway_Corners.shp airport_trajectories_shapefile_path = /aviation/Air_Trajectory.shp diff --git a/hermesv3_bu/sectors/aviation_sector.py b/hermesv3_bu/sectors/aviation_sector.py index 399efd3..a18631b 100755 --- a/hermesv3_bu/sectors/aviation_sector.py +++ b/hermesv3_bu/sectors/aviation_sector.py @@ -10,6 +10,8 @@ import geopandas as gpd from warnings import warn from hermesv3_bu.sectors.sector import Sector +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files, error_exit PHASE_TYPE = {'taxi_out': 'departure', 'pre-taxi_out': 'departure', 'takeoff': 'departure', 'climbout': 'departure', 'approach': 'arrival', 'taxi_in': 'arrival', 'post-taxi_in': 'arrival', 'landing': 'arrival', @@ -50,8 +52,8 @@ class AviationSector(Sector): created yet. :type auxiliary_dir: str - :param grid_shp: Shapefile with the grid horizontal distribution. - :type grid_shp: GeoDataFrame + :param grid: Grid object. + :type grid: Grid :param date_array: List of datetimes. :type date_array: list(datetime.datetime, ...) @@ -123,8 +125,16 @@ class AviationSector(Sector): file must contain the 'Specie' and 'MW' columns. :type molecular_weights_path: str """ + spent_time = timeit.default_timer() + logger.write_log('===== AVIATION SECTOR =====') + check_files( + [airport_shapefile_path, airport_runways_shapefile_path, airport_runways_corners_shapefile_path, + airport_trajectories_shapefile_path, operations_path, planes_path, times_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path] + + [os.path.join(ef_dir, PHASE_EF_FILE[phase]) for phase in PHASE_TYPE.keys()]) + super(AviationSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, None, weekly_profiles_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, @@ -295,7 +305,7 @@ class AviationSector(Sector): operations = operations.loc[operations['plane_id'].isin(self.plane_list), :] if len(operations) == 0: - raise NameError("The plane/s defined in the plane_list do not exist.") + error_exit("The plane/s defined in the plane_list do not exist.") operations = operations.loc[operations['airport_id'].isin(self.airport_list), :] operations.set_index(['airport_id', 'plane_id', 'operation'], inplace=True) operations.rename(columns={'1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, '10': 10, @@ -373,8 +383,8 @@ class AviationSector(Sector): shp_airport_list = list(set(conf_airport_list).intersection(shp_airport_list)) if len(shp_airport_list) == 0: - raise NameError("No airports intersect with the defined domain or the defined aiport/s in the " + - "airport_list do no exist ") + error_exit("No airports intersect with the defined domain or the defined aiport/s in the " + + "airport_list do no exist ") airports_with_operations = np.unique(pd.read_csv(operations_file, usecols=['airport_id']).values) @@ -391,9 +401,9 @@ class AviationSector(Sector): for i in range(self.comm.size)] for sublist in new_list: if len(sublist) == 0: - raise ValueError("ERROR: The selected number of processors is to high. " + - "The maximum number of processors accepted are {0}".format(max_len) + - "(Maximum number of airports included in the working domain") + error_exit("The selected number of processors is to high. " + + "The maximum number of processors accepted are {0}".format(max_len) + + "(Maximum number of airports included in the working domain") else: new_list = None diff --git a/hermesv3_bu/tools/checker.py b/hermesv3_bu/tools/checker.py index 27c2c0a..a2e06da 100644 --- a/hermesv3_bu/tools/checker.py +++ b/hermesv3_bu/tools/checker.py @@ -3,9 +3,10 @@ import os import sys from mpi4py import MPI +from warnings import warn -def check_file(file_path_list, warning=False): +def check_files(file_path_list, warning=False): if isinstance(file_path_list, str): file_path_list = [file_path_list] @@ -18,5 +19,16 @@ def check_file(file_path_list, warning=False): error_message = "*ERROR* (Rank {0}) File/s not found:".format(MPI.COMM_WORLD.Get_rank()) for file_path in files_not_found: error_message += "\n\t{0}".format(file_path) - print(error_message, file=sys.stderr) - MPI.COMM_WORLD.Abort() + + if warning: + warn(error_message.replace('ERROR', 'WARNING')) + else: + error_exit(error_message) + return True + + +def error_exit(error_message): + if not error_message[:7] == "*ERROR*": + error_message = "*ERROR* (Rank {0}) ".format(MPI.COMM_WORLD.Get_rank()) + error_message + print(error_message, file=sys.stderr) + MPI.COMM_WORLD.Abort() -- GitLab From 85eb5ae7b4dff10d4e30c51833e1defde8480ea7 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 11:09:21 +0200 Subject: [PATCH 03/16] - Shipping port: Checked files and raises --- conf/hermes.conf | 6 +++--- hermesv3_bu/sectors/shipping_port_sector.py | 15 ++++++++++----- hermesv3_bu/tools/checker.py | 1 + 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index b613126..3c43a9a 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -130,8 +130,8 @@ writing_processors = 1 # traffic_processors = 256 # traffic_area_processors = 1 -aviation_processors = 1 -shipping_port_processors = 0 +aviation_processors = 0 +shipping_port_processors = 1 livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 @@ -175,7 +175,7 @@ layer_thickness_dir = /esarchive/exp/monarch/a1wd/regional/hourly/layer_thicknes aviation_source_pollutants = nox_no2, co, hc, so2, pm10, pm25, co2 # airport_list = # plane_list = -airport_shapefile_path = /aviation/Airports.shpX +airport_shapefile_path = /aviation/Airports.shp airport_runways_shapefile_path = /aviation/Runways.shp airport_runways_corners_shapefile_path = /aviation/Runway_Corners.shp airport_trajectories_shapefile_path = /aviation/Air_Trajectory.shp diff --git a/hermesv3_bu/sectors/shipping_port_sector.py b/hermesv3_bu/sectors/shipping_port_sector.py index 0e316c2..88759a2 100755 --- a/hermesv3_bu/sectors/shipping_port_sector.py +++ b/hermesv3_bu/sectors/shipping_port_sector.py @@ -7,6 +7,8 @@ import numpy as np import timeit from hermesv3_bu.logger.log import Log from hermesv3_bu.io_server.io_shapefile import IoShapefile +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files, error_exit class ShippingPortSector(Sector): @@ -28,8 +30,8 @@ class ShippingPortSector(Sector): created yet. :type auxiliary_dir: str - :param grid_shp: Shapefile with the grid horizontal distribution. - :type grid_shp: GeoDataFrame + :param grid: Grid object. + :type grid: Grid :param date_array: List of datetimes. :type date_array: list(datetime.datetime, ...) @@ -82,6 +84,11 @@ class ShippingPortSector(Sector): spent_time = timeit.default_timer() logger.write_log('===== SHIPPING PORT SECTOR =====') + check_files( + [hoteling_shapefile_path, maneuvering_shapefile_path, engine_percent_path, tonnage_path, load_factor_path, + power_path, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, + speciation_profiles_path, molecular_weights_path, ef_dir]) + super(ShippingPortSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -111,7 +118,7 @@ class ShippingPortSector(Sector): port_shp = gpd.sjoin(port_shp, self.clip.shapefile.to_crs(port_shp.crs), how='inner', op='intersects') port_list = np.unique(port_shp['code'].values) if len(port_list) < self.comm.Get_size(): - raise ValueError("The chosen number of processors {0} exceeds the number of involved ports {1}.".format( + error_exit("The chosen number of processors {0} exceeds the number of involved ports {1}.".format( self.comm.Get_size(), len(port_list)) + " Set {0} at shipping_port_processors value.".format( len(port_list))) port_list = np.array_split(port_list, self.comm.Get_size()) @@ -120,8 +127,6 @@ class ShippingPortSector(Sector): port_list = self.comm.scatter(port_list, root=0) - if len(port_list) == 0: - raise ValueError("The number ") return list(port_list) def read_monthly_profiles(self, path): diff --git a/hermesv3_bu/tools/checker.py b/hermesv3_bu/tools/checker.py index a2e06da..e13b06d 100644 --- a/hermesv3_bu/tools/checker.py +++ b/hermesv3_bu/tools/checker.py @@ -22,6 +22,7 @@ def check_files(file_path_list, warning=False): if warning: warn(error_message.replace('ERROR', 'WARNING')) + return False else: error_exit(error_message) return True -- GitLab From 4860827824bd85a5e7db17564e2023d0ed0f3d72 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 11:34:42 +0200 Subject: [PATCH 04/16] - Livestock: Checked files and raises --- conf/hermes.conf | 4 ++-- hermesv3_bu/io_server/io_netcdf.py | 8 +++++--- hermesv3_bu/sectors/livestock_sector.py | 20 +++++++++++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 3c43a9a..12b9bff 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -131,8 +131,8 @@ writing_processors = 1 # traffic_area_processors = 1 aviation_processors = 0 -shipping_port_processors = 1 -livestock_processors = 0 +shipping_port_processors = 0 +livestock_processors = 1 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 diff --git a/hermesv3_bu/io_server/io_netcdf.py b/hermesv3_bu/io_server/io_netcdf.py index 01d0e83..48488aa 100755 --- a/hermesv3_bu/io_server/io_netcdf.py +++ b/hermesv3_bu/io_server/io_netcdf.py @@ -4,15 +4,16 @@ import sys import os from mpi4py import MPI from datetime import timedelta -from hermesv3_bu.io_server.io_server import IoServer import numpy as np import geopandas as gpd from netCDF4 import Dataset from shapely.geometry import Point from cf_units import num2date, CALENDAR_STANDARD - from geopandas import GeoDataFrame +from hermesv3_bu.io_server.io_server import IoServer +from hermesv3_bu.tools.checker import check_files, error_exit + class IoNetcdf(IoServer): def __init__(self, comm): @@ -42,6 +43,7 @@ class IoNetcdf(IoServer): :return: GeoDataframe with the data in the desired points. :rtype: geopandas.GeoDataframe """ + check_files(netcdf_path) nc = Dataset(netcdf_path, mode='r') lat_o = nc.variables['latitude'][:] lon_o = nc.variables['longitude'][:] @@ -106,7 +108,7 @@ class IoNetcdf(IoServer): path = os.path.join(netcdf_dir, '{0}_{1}{2}.nc'.format(var_name, date_array[0].year, str(date_array[0].month).zfill(2))) # self.logger.write_log('Getting temperature from {0}'.format(path), message_level=2) - + check_files(path) nc = Dataset(path, mode='r') lat_o = nc.variables['latitude'][:] lon_o = nc.variables['longitude'][:] diff --git a/hermesv3_bu/sectors/livestock_sector.py b/hermesv3_bu/sectors/livestock_sector.py index bede311..2cf1cc9 100755 --- a/hermesv3_bu/sectors/livestock_sector.py +++ b/hermesv3_bu/sectors/livestock_sector.py @@ -12,6 +12,8 @@ from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_raster import IoRaster from hermesv3_bu.io_server.io_netcdf import IoNetcdf +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files, error_exit # Constants for grassing daily factor estimation SIGMA = 60 @@ -43,8 +45,8 @@ class LivestockSector(Sector): ['nox_no', 'nh3', 'nmvoc', 'pm10', 'pm25'] :type source_pollutants: list - :param grid_shp: Shapefile that contains the destination grid. It must contains the 'FID' (cell num). - :type grid_shp: GeoPandas.GeoDataframe + :param grid: Grid object. + :type grid: Grid :param clip: Clip. :type clip: Clip @@ -153,6 +155,18 @@ class LivestockSector(Sector): """ spent_time = timeit.default_timer() logger.write_log('===== LIVESTOCK SECTOR =====') + check_files( + [gridded_livestock_path.replace('', animal) for animal in animal_list] + + [correction_split_factors_path.replace('', animal) for animal in animal_list] + + [temperature_dir, wind_speed_dir, + denominator_yearly_factor_dir, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, + speciation_map_path, speciation_profiles_path, molecular_weights_path, nut_shapefile_path] + + [os.path.join(ef_dir, ef_file) for ef_file in + ['{0}.csv'.format(pol) for pol in source_pollutants if pol not in ['pm10', 'pm25']]]) + for pol in source_pollutants: + if pol in ['pm10', 'pm25']: + check_files(os.path.join(ef_dir, 'pm.csv')) + super(LivestockSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -722,7 +736,7 @@ class LivestockSector(Sector): (animals_df[animal['Code']] * animals_df['FD_housing_closed']).multiply( animal['EF_housing']) else: - raise KeyError('Animal {0} not found on the nh3 emission factors file.'.format(animal.Code)) + error_exit('Animal {0} not found on the nh3 emission factors file.'.format(animal.Code)) # Storage emissions out_df.loc[:, out_p] += \ (animals_df[animal['Code']] * animals_df['FD_storage']).multiply(animal['EF_yarding']) -- GitLab From 32883896d657ec750499fb25102100f0846775f2 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 11:39:00 +0200 Subject: [PATCH 05/16] - IO Servers: Checked files and raises --- hermesv3_bu/io_server/io_netcdf.py | 2 +- hermesv3_bu/io_server/io_raster.py | 9 +++------ hermesv3_bu/io_server/io_shapefile.py | 4 +++- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/hermesv3_bu/io_server/io_netcdf.py b/hermesv3_bu/io_server/io_netcdf.py index 48488aa..6c8480d 100755 --- a/hermesv3_bu/io_server/io_netcdf.py +++ b/hermesv3_bu/io_server/io_netcdf.py @@ -12,7 +12,7 @@ from cf_units import num2date, CALENDAR_STANDARD from geopandas import GeoDataFrame from hermesv3_bu.io_server.io_server import IoServer -from hermesv3_bu.tools.checker import check_files, error_exit +from hermesv3_bu.tools.checker import check_files class IoNetcdf(IoServer): diff --git a/hermesv3_bu/io_server/io_raster.py b/hermesv3_bu/io_server/io_raster.py index 030c8b8..9a9d73b 100755 --- a/hermesv3_bu/io_server/io_raster.py +++ b/hermesv3_bu/io_server/io_raster.py @@ -1,19 +1,16 @@ #!/usr/bin/env python -import sys import os import timeit -from warnings import warn from mpi4py import MPI import rasterio from rasterio.mask import mask import geopandas as gpd -import pandas as pd import numpy as np from shapely.geometry import Polygon - from hermesv3_bu.io_server.io_server import IoServer +from hermesv3_bu.tools.checker import check_files class IoRaster(IoServer): @@ -49,7 +46,7 @@ class IoRaster(IoServer): Function to parse features from GeoDataFrame in such a manner that rasterio wants them""" import json return [json.loads(gdf.to_json())['features'][0]['geometry']] - + check_files([raster_path, shape_path]) data = rasterio.open(raster_path) geo = gpd.read_file(shape_path) if len(geo) > 1: @@ -106,7 +103,7 @@ class IoRaster(IoServer): Function to parse features from GeoDataFrame in such a manner that rasterio wants them""" import json return [json.loads(gdf.to_json())['features'][0]['geometry']] - + check_files(raster_path) data = rasterio.open(raster_path) if len(geo) > 1: diff --git a/hermesv3_bu/io_server/io_shapefile.py b/hermesv3_bu/io_server/io_shapefile.py index f2c89d6..d4a5df1 100755 --- a/hermesv3_bu/io_server/io_shapefile.py +++ b/hermesv3_bu/io_server/io_shapefile.py @@ -10,6 +10,7 @@ import geopandas as gpd from mpi4py import MPI from hermesv3_bu.io_server.io_server import IoServer +from hermesv3_bu.tools.checker import check_files class IoShapefile(IoServer): @@ -59,13 +60,14 @@ class IoShapefile(IoServer): return True def read_shapefile_serial(self, path): - + check_files(path) gdf = gpd.read_file(path) return gdf def read_shapefile(self, path, rank=0): if self.comm.Get_rank() == rank: + check_files(path) gdf = gpd.read_file(path) gdf = np.array_split(gdf, self.comm.Get_size()) else: -- GitLab From cf51749404d962c45ccfc62ab79b9112b4672b5b Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 11:45:19 +0200 Subject: [PATCH 06/16] - Crop operations: Checked files and raises --- conf/hermes.conf | 4 ++-- .../sectors/agricultural_crop_operations_sector.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 12b9bff..8ef4037 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -132,8 +132,8 @@ writing_processors = 1 aviation_processors = 0 shipping_port_processors = 0 -livestock_processors = 1 -crop_operations_processors = 0 +livestock_processors = 0 +crop_operations_processors = 1 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 residential_processors = 0 diff --git a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py index f456687..9edb2a9 100755 --- a/hermesv3_bu/sectors/agricultural_crop_operations_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_operations_sector.py @@ -8,6 +8,8 @@ import numpy as np from hermesv3_bu.sectors.agricultural_sector import AgriculturalSector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.logger.log import Log +from hermesv3_bu.grids.grid import Grid +from hermesv3_bu.tools.checker import check_files class AgriculturalCropOperationsSector(AgriculturalSector): @@ -21,8 +23,8 @@ class AgriculturalCropOperationsSector(AgriculturalSector): not created yet. :type auxiliary_dir: str - :param grid_shp: Shapefile that contains the destination grid. It must contains the 'FID' (cell num). - :type grid_shp: GeoPandas.GeoDataframe + :param grid: Grid object. + :type grid: Grid :param clip: Path to the shapefile that contains the region of interest. :type clip: str @@ -89,6 +91,13 @@ class AgriculturalCropOperationsSector(AgriculturalSector): """ spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL CROP OPERATIONS SECTOR =====') + + check_files( + [nut_shapefile_path, land_uses_path, monthly_profiles_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path, + landuse_by_nut, crop_by_nut, crop_from_landuse_path] + + [os.path.join(ef_dir, ef_file) for ef_file in ['{0}.csv'.format(pol) for pol in source_pollutants]]) + super(AgriculturalCropOperationsSector, self).__init__( comm_agr, comm, logger, auxiliary_dir, grid, clip, date_array, nut_shapefile_path, source_pollutants, vertical_levels, crop_list, land_uses_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, ef_dir, -- GitLab From d21d6044e1478ba578a3f72a272f5524d3d108e8 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 12:23:12 +0200 Subject: [PATCH 07/16] - Crop fertilizers: Checked files and raises --- conf/hermes.conf | 6 +++--- .../sectors/agricultural_crop_fertilizers_sector.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 8ef4037..b0ddcd7 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -133,8 +133,8 @@ writing_processors = 1 aviation_processors = 0 shipping_port_processors = 0 livestock_processors = 0 -crop_operations_processors = 1 -crop_fertilizers_processors = 0 +crop_operations_processors = 0 +crop_fertilizers_processors = 1 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 @@ -234,7 +234,7 @@ crop_operations_speciation_profiles = /profiles/speciation/agricultur [AGRICULTURAL_CROP_FERTILIZERS] crop_fertilizers_source_pollutants = nh3 crop_fertilizers_list = alfalfa, almond, apple, apricot, barley, cherry, cotton, fig, grape, lemonlime, maize, melonetc, oats, olive, orange, pea, peachetc, pear, potato, rice, rye, sunflower, tangetc, tomato, triticale, vetch, watermelon, wheat -cultivated_ratio = /agriculture/agriculture_fertilizers/fertilized_cultivated_ratio.csv +cultivated_ratio = /agriculture/agriculture_fertilizers/fertilized_cultivated_ratio.csvZ fertilizers_rate = /agriculture/agriculture_fertilizers/fertilizer_ratio.csv crop_f_parameter = /agriculture/agriculture_fertilizers/ef/crops_f_parameter.csv crop_f_fertilizers = /agriculture/agriculture_fertilizers/ef/f_fertilizers.csv diff --git a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py index b291236..27bb100 100755 --- a/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py +++ b/hermesv3_bu/sectors/agricultural_crop_fertilizers_sector.py @@ -10,8 +10,9 @@ from hermesv3_bu.io_server.io_raster import IoRaster from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_netcdf import IoNetcdf from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files -formula = True +FORMULA = True class AgriculturalCropFertilizersSector(AgriculturalSector): @@ -23,6 +24,11 @@ class AgriculturalCropFertilizersSector(AgriculturalSector): crop_growing_degree_day_path): spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL CROP FERTILIZERS SECTOR =====') + check_files( + [nut_shapefile, land_uses_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, + molecular_weights_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, cultivated_ratio, + fertilizer_rate, crop_f_parameter, crop_f_fertilizers, gridded_ph, gridded_cec, crop_calendar, + temperature_path, wind_speed_path]) super(AgriculturalCropFertilizersSector, self).__init__( comm_agr, comm, logger, auxiliary_dir, grid, clip, date_array, nut_shapefile, source_pollutants, vertical_levels, crop_list, land_uses_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, None, None, @@ -93,7 +99,7 @@ class AgriculturalCropFertilizersSector(AgriculturalSector): for crop in self.crop_list: crop_ef = self.gridded_constants.loc[:, ['geometry', 'nut_code']].copy() # f_ph - if formula: + if FORMULA: # After Zhang et al. (2018) crop_ef['f_ph'] = (0.067 * self.gridded_constants['ph'] ** 2) - \ (0.69 * self.gridded_constants['ph']) + 0.68 -- GitLab From a6e1c18625b85ce2eb8885cd51d0a0a12f2f4826 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 12:26:26 +0200 Subject: [PATCH 08/16] - Agricultural machinery: Checked files and raises --- conf/hermes.conf | 6 +++--- hermesv3_bu/sectors/agricultural_machinery_sector.py | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index b0ddcd7..4aa0123 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -134,8 +134,8 @@ aviation_processors = 0 shipping_port_processors = 0 livestock_processors = 0 crop_operations_processors = 0 -crop_fertilizers_processors = 1 -agricultural_machinery_processors = 0 +crop_fertilizers_processors = 0 +agricultural_machinery_processors = 1 residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 @@ -234,7 +234,7 @@ crop_operations_speciation_profiles = /profiles/speciation/agricultur [AGRICULTURAL_CROP_FERTILIZERS] crop_fertilizers_source_pollutants = nh3 crop_fertilizers_list = alfalfa, almond, apple, apricot, barley, cherry, cotton, fig, grape, lemonlime, maize, melonetc, oats, olive, orange, pea, peachetc, pear, potato, rice, rye, sunflower, tangetc, tomato, triticale, vetch, watermelon, wheat -cultivated_ratio = /agriculture/agriculture_fertilizers/fertilized_cultivated_ratio.csvZ +cultivated_ratio = /agriculture/agriculture_fertilizers/fertilized_cultivated_ratio.csv fertilizers_rate = /agriculture/agriculture_fertilizers/fertilizer_ratio.csv crop_f_parameter = /agriculture/agriculture_fertilizers/ef/crops_f_parameter.csv crop_f_fertilizers = /agriculture/agriculture_fertilizers/ef/f_fertilizers.csv diff --git a/hermesv3_bu/sectors/agricultural_machinery_sector.py b/hermesv3_bu/sectors/agricultural_machinery_sector.py index 7fb03bf..56d848d 100755 --- a/hermesv3_bu/sectors/agricultural_machinery_sector.py +++ b/hermesv3_bu/sectors/agricultural_machinery_sector.py @@ -11,7 +11,7 @@ import numpy as np from hermesv3_bu.sectors.agricultural_sector import AgriculturalSector from hermesv3_bu.io_server.io_shapefile import IoShapefile -from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files class AgriculturalMachinerySector(AgriculturalSector): @@ -25,6 +25,12 @@ class AgriculturalMachinerySector(AgriculturalSector): spent_time = timeit.default_timer() logger.write_log('===== AGRICULTURAL MACHINERY SECTOR =====') + check_files( + [nut_shapefile, land_uses_path, ef_files_dir, monthly_profiles_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path, + landuse_by_nut, crop_by_nut, crop_from_landuse_path, machinery_distibution_nut_shapefile_path, + deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, vehicle_workhours_path, + vehicle_power_path, crop_machinery_nuts3]) super(AgriculturalMachinerySector, self).__init__( comm_agr, comm, logger, auxiliary_dir, grid, clip, date_array, nut_shapefile, source_pollutants, vertical_levels, crop_list, land_uses_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, -- GitLab From ae12a01d13eaea27e124c4deb6abcffeaeb4d499 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 12:36:00 +0200 Subject: [PATCH 09/16] - Residential: Checked files and raises --- conf/hermes.conf | 6 +++--- hermesv3_bu/sectors/residential_sector.py | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 4aa0123..b881f9a 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -135,8 +135,8 @@ shipping_port_processors = 0 livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 -agricultural_machinery_processors = 1 -residential_processors = 0 +agricultural_machinery_processors = 0 +residential_processors = 1 recreational_boats_processors = 0 point_sources_processors = 0 traffic_processors = 0 @@ -282,7 +282,7 @@ residential_speciation_profiles = /profiles/speciation/residential/sp [RECREATIONAL_BOATS] recreational_boats_source_pollutants = nox_no2,so2,nmvoc,co,nh3,pm10,pm25,co2,ch4 recreational_boats_list = YB_001,YB_002,SB_001,SB_002,SP_001,SP_002,OB_001,OB_002,WS_001,WS_002,YB_003,SB_003,SP_004,SP_005,OB_002,WS_003,MB_001,MB_002,MB_003,MB_004,MB_005,MB_006,MS_001,MS_002,SB_004,SB_005 -recreational_boats_density_map = /recreational_boats/recreation_boats_area.tif +recreational_boats_density_map = /recreational_boats/recreation_boats_area.tifx recreational_boats_by_type = /recreational_boats/recreation_boats.csv recreational_boats_ef_path = /recreational_boats/ef_recreation_boats.csv recreational_boats_monthly_profiles = /profiles/temporal/recreational_boats/monthly_profiles.csv diff --git a/hermesv3_bu/sectors/residential_sector.py b/hermesv3_bu/sectors/residential_sector.py index 17f6904..d098775 100755 --- a/hermesv3_bu/sectors/residential_sector.py +++ b/hermesv3_bu/sectors/residential_sector.py @@ -12,7 +12,7 @@ from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_raster import IoRaster from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_netcdf import IoNetcdf -from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files class ResidentialSector(Sector): @@ -23,7 +23,12 @@ class ResidentialSector(Sector): heating_degree_day_path, temperature_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path): spent_time = timeit.default_timer() - + logger.write_log('===== RESIDENTIAL COMBUSTION SECTOR =====') + check_files( + [prov_shapefile, ccaa_shapefile, population_density_map, population_type_map, population_type_nuts2, + population_type_nuts3, energy_consumption_nuts3, energy_consumption_nuts2, residential_spatial_proxies, + residential_ef_files_path, temperature_path, hourly_profiles_path, speciation_map_path, + speciation_profiles_path, molecular_weights_path]) super(ResidentialSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, None, None, hourly_profiles_path, speciation_map_path, -- GitLab From 1b66dc7775a43d9fcbd18eef09af9d0b08cb5066 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 12:38:38 +0200 Subject: [PATCH 10/16] - Recreational boats: Checked files and raises --- conf/hermes.conf | 8 ++++---- hermesv3_bu/sectors/recreational_boats_sector.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index b881f9a..555227b 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -136,8 +136,8 @@ livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 -residential_processors = 1 -recreational_boats_processors = 0 +residential_processors = 0 +recreational_boats_processors = 1 point_sources_processors = 0 traffic_processors = 0 traffic_area_processors = 0 @@ -282,7 +282,7 @@ residential_speciation_profiles = /profiles/speciation/residential/sp [RECREATIONAL_BOATS] recreational_boats_source_pollutants = nox_no2,so2,nmvoc,co,nh3,pm10,pm25,co2,ch4 recreational_boats_list = YB_001,YB_002,SB_001,SB_002,SP_001,SP_002,OB_001,OB_002,WS_001,WS_002,YB_003,SB_003,SP_004,SP_005,OB_002,WS_003,MB_001,MB_002,MB_003,MB_004,MB_005,MB_006,MS_001,MS_002,SB_004,SB_005 -recreational_boats_density_map = /recreational_boats/recreation_boats_area.tifx +recreational_boats_density_map = /recreational_boats/recreation_boats_area.tif recreational_boats_by_type = /recreational_boats/recreation_boats.csv recreational_boats_ef_path = /recreational_boats/ef_recreation_boats.csv recreational_boats_monthly_profiles = /profiles/temporal/recreational_boats/monthly_profiles.csv @@ -294,7 +294,7 @@ recreational_boats_speciation_profiles = /profiles/speciation/recreat point_source_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25,ch4,n2o,co2 plume_rise = True # point_source_snaps = 09 -point_source_catalog = /point_sources/Maestra_focos_2015_plume_rise.shp +point_source_catalog = /point_sources/Maestra_focos_2015_plume_rise.shpx point_source_monthly_profiles = /profiles/temporal/point_sources/monthly_profiles.csv point_source_weekly_profiles = /profiles/temporal/point_sources/weekly_profiles.csv point_source_hourly_profiles = /profiles/temporal/point_sources/hourly_profiles.csv diff --git a/hermesv3_bu/sectors/recreational_boats_sector.py b/hermesv3_bu/sectors/recreational_boats_sector.py index 462c7d6..4b7c5c7 100755 --- a/hermesv3_bu/sectors/recreational_boats_sector.py +++ b/hermesv3_bu/sectors/recreational_boats_sector.py @@ -10,7 +10,7 @@ import geopandas as gpd from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_raster import IoRaster -from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files class RecreationalBoatsSector(Sector): @@ -19,7 +19,10 @@ class RecreationalBoatsSector(Sector): weekly_profiles_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path): spent_time = timeit.default_timer() - + logger.write_log('===== RECREATIONAL BOATS SECTOR =====') + check_files( + [density_map_path, boats_data_path, ef_file_path, monthly_profiles_path, weekly_profiles_path, + hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path]) super(RecreationalBoatsSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, -- GitLab From 1ddd82ac1e9e1a6e00653ef9f4ac26faf5d01b66 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 12:46:44 +0200 Subject: [PATCH 11/16] - Point sources: Checked files and raises --- conf/hermes.conf | 6 +++--- hermesv3_bu/sectors/point_source_sector.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 555227b..61a70c7 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -137,10 +137,10 @@ crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 residential_processors = 0 -recreational_boats_processors = 1 +recreational_boats_processors = 0 point_sources_processors = 0 traffic_processors = 0 -traffic_area_processors = 0 +traffic_area_processors = 1 [SHAPEFILES] @@ -294,7 +294,7 @@ recreational_boats_speciation_profiles = /profiles/speciation/recreat point_source_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25,ch4,n2o,co2 plume_rise = True # point_source_snaps = 09 -point_source_catalog = /point_sources/Maestra_focos_2015_plume_rise.shpx +point_source_catalog = /point_sources/Maestra_focos_2015_plume_rise.shp point_source_monthly_profiles = /profiles/temporal/point_sources/monthly_profiles.csv point_source_weekly_profiles = /profiles/temporal/point_sources/weekly_profiles.csv point_source_hourly_profiles = /profiles/temporal/point_sources/hourly_profiles.csv diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index b9555a9..e3c2af6 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -10,6 +10,7 @@ from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile # from hermesv3_bu.io_server.io_netcdf import IoNetcdf from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import check_files, error_exit INTERPOLATION_TYPE = 'linear' # GRAVITI m/s-2 @@ -22,8 +23,8 @@ class PointSourceSector(Sector): """ Class to calculate the Point Source emissions - :param grid_shp: Grid of the destination domain - :type grid_shp: Grid + :param grid: Grid of the destination domain + :type grid: Grid :param catalog_path: Path to the fine that contains all the information for each point source. :type catalog_path: str @@ -51,7 +52,10 @@ class PointSourceSector(Sector): speciation_map_path, speciation_profiles_path, sector_list, measured_emission_path, molecular_weights_path, plume_rise=False, plume_rise_pahts=None): spent_time = timeit.default_timer() - + logger.write_log('===== POINT SOURCES SECTOR =====') + check_files( + [catalog_path, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, + speciation_profiles_path]) super(PointSourceSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, @@ -782,7 +786,7 @@ class PointSourceSector(Sector): try: test.set_index(x.index, inplace=True) except ValueError: - raise IOError('No measured emissions for the selected dates: {0}'.format(x.values)) + error_exit('No measured emissions for the selected dates: {0}'.format(x.values)) return test[pollutant] -- GitLab From 93253fc9a1a94167590e709482ca4421f91f5136 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 13:01:48 +0200 Subject: [PATCH 12/16] - Traffic area: Checked files and raises --- conf/hermes.conf | 2 +- hermesv3_bu/sectors/agricultural_machinery_sector.py | 10 +++++----- hermesv3_bu/sectors/traffic_area_sector.py | 12 ++++++++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 61a70c7..243d8ea 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -334,7 +334,7 @@ traffic_speciation_profile_resuspension = /profiles/speciation/traffi [TRAFFIC AREA SECTOR] traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 do_evaporative = 1 -traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csv +traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csvx population_nuts3 = /traffic_area/population_by_mun.csv traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_base.csv traffic_area_evaporative_ef_file = /traffic_area/ef/evaporative_nmvoc.csv diff --git a/hermesv3_bu/sectors/agricultural_machinery_sector.py b/hermesv3_bu/sectors/agricultural_machinery_sector.py index 56d848d..25f9659 100755 --- a/hermesv3_bu/sectors/agricultural_machinery_sector.py +++ b/hermesv3_bu/sectors/agricultural_machinery_sector.py @@ -19,7 +19,7 @@ class AgriculturalMachinerySector(AgriculturalSector): vertical_levels, crop_list, nut_shapefile, machinery_list, land_uses_path, ef_files_dir, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, - machinery_distibution_nut_shapefile_path, deterioration_factor_path, load_factor_path, + machinery_distribution_nut_shapefile_path, deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, vehicle_workhours_path, vehicle_power_path, crop_machinery_nuts3): spent_time = timeit.default_timer() @@ -28,9 +28,9 @@ class AgriculturalMachinerySector(AgriculturalSector): check_files( [nut_shapefile, land_uses_path, ef_files_dir, monthly_profiles_path, weekly_profiles_path, hourly_profiles_path, speciation_map_path, speciation_profiles_path, molecular_weights_path, - landuse_by_nut, crop_by_nut, crop_from_landuse_path, machinery_distibution_nut_shapefile_path, - deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, vehicle_workhours_path, - vehicle_power_path, crop_machinery_nuts3]) + landuse_by_nut, crop_by_nut, crop_from_landuse_path, machinery_distribution_nut_shapefile_path, + deterioration_factor_path, load_factor_path, vehicle_ratio_path, vehicle_units_path, + vehicle_workhours_path, vehicle_power_path, crop_machinery_nuts3]) super(AgriculturalMachinerySector, self).__init__( comm_agr, comm, logger, auxiliary_dir, grid, clip, date_array, nut_shapefile, source_pollutants, vertical_levels, crop_list, land_uses_path, landuse_by_nut, crop_by_nut, crop_from_landuse_path, @@ -41,7 +41,7 @@ class AgriculturalMachinerySector(AgriculturalSector): self.crop_machinery_nuts3 = self.read_profiles(crop_machinery_nuts3) self.crop_distribution = self.get_crop_distribution_by_nut( - self.crop_distribution, machinery_distibution_nut_shapefile_path, nut_code='nuts3_id') + self.crop_distribution, machinery_distribution_nut_shapefile_path, nut_code='nuts3_id') self.months = self.get_date_array_by_month() diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 3047954..237dde6 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -9,6 +9,8 @@ import numpy as np from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_shapefile import IoShapefile from hermesv3_bu.io_server.io_netcdf import IoNetcdf +from hermesv3_bu.tools.checker import check_files, error_exit + pmc_list = ['pmc', 'PMC'] @@ -22,7 +24,13 @@ class TrafficAreaSector(Sector): small_cities_monthly_profile, small_cities_weekly_profile, small_cities_hourly_profile): spent_time = timeit.default_timer() logger.write_log('===== TRAFFIC AREA SECTOR =====') - + if do_evaporative: + check_files([population_tif_path, speciation_map_path, molecular_weights_path, + nuts_shapefile, gasoline_path, total_pop_by_prov]) + if do_small_cities: + check_files([population_tif_path, speciation_map_path, molecular_weights_path, + small_cities_shp, speciation_profiles_small_cities, small_cities_ef_file, + small_cities_monthly_profile, small_cities_weekly_profile, small_cities_hourly_profile]) super(TrafficAreaSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, None, None, None, speciation_map_path, None, molecular_weights_path) @@ -456,7 +464,7 @@ class TrafficAreaSector(Sector): elif self.do_small_cities: dataset = self.small_cities else: - raise ValueError('No traffic area emission selected. do_evaporative and do_small_cities are False') + error_exit('No traffic area emission selected. do_evaporative and do_small_cities are False') dataset['layer'] = 0 dataset = dataset.groupby(['FID', 'layer', 'tstep']).sum() -- GitLab From 8ba365bda1f049714893cfda5ee5e9beb7f80c97 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 13:10:06 +0200 Subject: [PATCH 13/16] - Traffic area: Checked files and raises --- conf/hermes.conf | 2 +- hermesv3_bu/sectors/traffic_area_sector.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 243d8ea..61a70c7 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -334,7 +334,7 @@ traffic_speciation_profile_resuspension = /profiles/speciation/traffi [TRAFFIC AREA SECTOR] traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 do_evaporative = 1 -traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csvx +traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csv population_nuts3 = /traffic_area/population_by_mun.csv traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_base.csv traffic_area_evaporative_ef_file = /traffic_area/ef/evaporative_nmvoc.csv diff --git a/hermesv3_bu/sectors/traffic_area_sector.py b/hermesv3_bu/sectors/traffic_area_sector.py index 237dde6..2104525 100755 --- a/hermesv3_bu/sectors/traffic_area_sector.py +++ b/hermesv3_bu/sectors/traffic_area_sector.py @@ -26,7 +26,8 @@ class TrafficAreaSector(Sector): logger.write_log('===== TRAFFIC AREA SECTOR =====') if do_evaporative: check_files([population_tif_path, speciation_map_path, molecular_weights_path, - nuts_shapefile, gasoline_path, total_pop_by_prov]) + gasoline_path, total_pop_by_prov, nuts_shapefile, speciation_profiles_evaporative, + evaporative_ef_file, temperature_dir]) if do_small_cities: check_files([population_tif_path, speciation_map_path, molecular_weights_path, small_cities_shp, speciation_profiles_small_cities, small_cities_ef_file, -- GitLab From f98c72c804b93f34868bf0783393ba9d642c412a Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 13:49:36 +0200 Subject: [PATCH 14/16] - Traffic: Checked files and raises --- conf/hermes.conf | 4 +- hermesv3_bu/sectors/traffic_sector.py | 132 ++++++++++++++++---------- 2 files changed, 86 insertions(+), 50 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 61a70c7..0a9c137 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -139,8 +139,8 @@ agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 -traffic_processors = 0 -traffic_area_processors = 1 +traffic_processors = 1 +traffic_area_processors = 0 [SHAPEFILES] diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 41fcb2f..67a82b3 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -13,6 +13,7 @@ import warnings from hermesv3_bu.logger.log import Log from hermesv3_bu.sectors.sector import Sector from hermesv3_bu.io_server.io_netcdf import IoNetcdf +from hermesv3_bu.tools.checker import check_files, error_exit from ctypes import cdll, CDLL cdll.LoadLibrary("libc.so.6") @@ -55,6 +56,45 @@ class TrafficSector(Sector): spent_time = timeit.default_timer() logger.write_log('===== TRAFFIC SECTOR =====') + if do_hot: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, hot_cold_speciation] + + [os.path.join(ef_common_path, "hot_{0}.csv".format(pol)) for pol in source_pollutants]) + if do_cold: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, hot_cold_speciation, + temp_common_path] + + [os.path.join(ef_common_path, "cold_{0}.csv".format(pol)) for pol in source_pollutants]) + if do_tyre_wear: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, tyre_speciation] + + [os.path.join(ef_common_path, "tyre_{0}.csv".format(pol)) for pol in ['pm']]) + if do_road_wear: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, road_speciation] + + [os.path.join(ef_common_path, "road_{0}.csv".format(pol)) for pol in ['pm']]) + if do_brake_wear: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, brake_speciation] + + [os.path.join(ef_common_path, "brake_{0}.csv".format(pol)) for pol in ['pm']]) + if do_resuspension: + check_files( + [road_link_path, fleet_compo_path, speed_hourly_path, monthly_profiles_path, weekly_profiles_path, + hourly_mean_profiles_path, hourly_weekday_profiles_path, hourly_saturday_profiles_path, + hourly_sunday_profiles_path, speciation_map_path, molecular_weights_path, resuspension_speciation] + + [os.path.join(ef_common_path, "resuspension_{0}.csv".format(pol)) for pol in ['pm']]) + if resuspension_correction: + check_files(precipitation_path) super(TrafficSector, self).__init__( comm, logger, auxiliary_dir, grid, clip, date_array, source_pollutants, vertical_levels, monthly_profiles_path, weekly_profiles_path, None, speciation_map_path, None, molecular_weights_path) @@ -121,24 +161,24 @@ class TrafficSector(Sector): speed_res = links_speed - speed if len(speed_res) > 0: - raise ValueError("The following speed profile IDs reported in the road links shapefile do not appear " + - "in the hourly speed profiles file. {0}".format(speed_res)) + error_exit("The following speed profile IDs reported in the road links shapefile do not appear " + + "in the hourly speed profiles file. {0}".format(speed_res)) # Checking monthly profiles IDs links_month = set(np.unique(self.road_links['aadt_m_mn'].dropna().values)) month = set(self.monthly_profiles.index.values) month_res = links_month - month if len(month_res) > 0: - raise ValueError("The following monthly profile IDs reported in the road links shapefile do not appear " + - "in the monthly profiles file. {0}".format(month_res)) + error_exit("The following monthly profile IDs reported in the road links shapefile do not appear " + + "in the monthly profiles file. {0}".format(month_res)) # Checking weekly profiles IDs links_week = set(np.unique(self.road_links['aadt_week'].dropna().values)) week = set(self.weekly_profiles.index.values) week_res = links_week - week if len(week_res) > 0: - raise ValueError("The following weekly profile IDs reported in the road links shapefile do not appear " + - "in the weekly profiles file. {0}".format(week_res)) + error_exit("The following weekly profile IDs reported in the road links shapefile do not appear " + + "in the weekly profiles file. {0}".format(week_res)) # Checking hourly profiles IDs links_hour = set(np.unique(np.concatenate([ @@ -150,8 +190,8 @@ class TrafficSector(Sector): hour = set(self.hourly_profiles.index.values) hour_res = links_hour - hour if len(hour_res) > 0: - raise ValueError("The following hourly profile IDs reported in the road links shapefile do not appear " + - "in the hourly profiles file. {0}".format(hour_res)) + error_exit("The following hourly profile IDs reported in the road links shapefile do not appear " + + "in the hourly profiles file. {0}".format(hour_res)) self.logger.write_time_log('TrafficSector', 'check_profiles', timeit.default_timer() - spent_time) @@ -357,7 +397,7 @@ class TrafficSector(Sector): # Check if percents are ok if len(df[df['PcLight'] < 0]) is not 0: - raise ValueError('ERROR: PcLight < 0') + error_exit('PcLight < 0') if self.write_rline: self.write_rline_roadlinks(df) @@ -389,7 +429,7 @@ class TrafficSector(Sector): # Pollutants different to NH3 if pollutant_name != 'nh3': - del df['Copert_V_name'] + df.drop(columns=['Copert_V_name'], inplace=True) # For hot emission factors if emission_type == 'hot': @@ -398,7 +438,7 @@ class TrafficSector(Sector): df.loc[df['Technology'].isnull(), 'Technology'] = '' df = df[df['Technology'] != 'EGR'] - del df['Technology'], df['Load'] + df.drop(columns=['Technology', 'Load'], inplace=True) # Split the EF file into small DataFrames divided by column Road.Slope and Mode restrictions. df_code_slope_road = df[df['Road.Slope'].notnull() & df['Mode'].notnull()] @@ -409,7 +449,7 @@ class TrafficSector(Sector): # Checks that the splited DataFrames contain the full DataFrame if (len(df_code_slope_road) + len(df_code_slope) + len(df_code_road) + len(df_code)) != len(df): # TODO check that error - raise ValueError('ERROR in blablavbla') + error_exit('ERROR in blablavbla') return df_code_slope_road, df_code_slope, df_code_road, df_code elif emission_type == 'cold' or emission_type == 'tyre' or emission_type == 'road' or \ @@ -417,7 +457,7 @@ class TrafficSector(Sector): return df # NH3 pollutant else: - del df['Copert_V_name'] + df.drop(columns=['Copert_V_name'], inplace=True) # Specific case for cold NH3 emission factors that needs the hot emission factors and the cold ones. if emission_type == 'cold': df_hot = self.read_ef('hot', pollutant_name) @@ -425,8 +465,7 @@ class TrafficSector(Sector): df = df.merge(df_hot, left_on=['Code', 'Mode'], right_on=['Code_hot', 'Mode_hot'], how='left') - - del df['Cmileage_hot'], df['Mode_hot'], df['Code_hot'] + df.drop(columns=['Cmileage_hot', 'Mode_hot', 'Code_hot'], inplace=True) return df @@ -569,7 +608,7 @@ class TrafficSector(Sector): elif weekday == 6: return 'aadt_h_sun' else: - raise KeyError('ERROR: Weekday not found') + error_exit('Weekday not found') # Monthly factor x = pd.merge(x, self.monthly_profiles, left_on='aadt_m_mn', right_index=True, how='left') @@ -650,7 +689,7 @@ class TrafficSector(Sector): try: fleet = self.fleet_compo[['Code', 'Class', zone]] except KeyError as e: - raise KeyError(e.message + ' of the fleet_compo file') + error_exit(e.message + ' of the fleet_compo file') fleet.columns = ['Fleet_Code', 'Fleet_Class', 'Fleet_value'] fleet = fleet[fleet['Fleet_value'] > 0] @@ -690,7 +729,7 @@ class TrafficSector(Sector): expanded_aux = expanded_aux.merge(ef_code_road, left_on=['Fleet_Code', 'Road_type'], right_on=['Code', 'Mode'], how='inner') - del expanded_aux['Code'], expanded_aux['Mode'] + expanded_aux.drop(columns=['Code', 'Mode'], inplace=True) # Warnings and Errors original_ef_profile = np.unique(self.expanded.index.get_level_values('Fleet_Code')) @@ -703,7 +742,7 @@ class TrafficSector(Sector): resta_1)) warnings.warn('Exists some fleet codes that not appear on the EF file: {0}'.format(resta_1), Warning) if len(resta_2) > 0: - raise ImportError('Exists some fleet codes duplicateds on the EF file: {0}'.format(resta_2)) + error_exit('Exists some fleet codes duplicated on the EF file: {0}'.format(resta_2)) m_corr = self.read_mcorr_file(pollutant) if m_corr is not None: @@ -777,13 +816,10 @@ class TrafficSector(Sector): spent_time = timeit.default_timer() cold_links = self.road_links.copy().reset_index() - - del cold_links['aadt'], cold_links['PcHeavy'], cold_links['PcMoto'], cold_links['PcMoped'], cold_links['sp_wd'] - del cold_links['sp_we'], cold_links['sp_hour_su'], cold_links['sp_hour_mo'], cold_links['sp_hour_tu'] - del cold_links['sp_hour_we'], cold_links['sp_hour_th'], cold_links['sp_hour_fr'], cold_links['sp_hour_sa'] - del cold_links['Road_type'], cold_links['aadt_m_mn'], cold_links['aadt_h_mn'], cold_links['aadt_h_wd'] - del cold_links['aadt_h_sat'], cold_links['aadt_h_sun'], cold_links['aadt_week'], cold_links['fleet_comp'] - del cold_links['road_grad'], cold_links['PcLight'], cold_links['start_date'] + cold_links.drop(columns=['aadt', 'PcHeavy', 'PcMoto', 'PcMoped', 'sp_wd', 'sp_we', 'sp_hour_su', 'sp_hour_mo', + 'sp_hour_tu', 'sp_hour_we', 'sp_hour_th', 'sp_hour_fr', 'sp_hour_sa', 'Road_type', + 'aadt_m_mn', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun', 'aadt_week', + 'fleet_comp', 'road_grad', 'PcLight', 'start_date'], inplace=True) libc.malloc_trim(0) cold_links['centroid'] = cold_links['geometry'].centroid @@ -801,11 +837,12 @@ class TrafficSector(Sector): unary_union = temperature.unary_union cold_links['REC'] = cold_links.apply(self.nearest, geom_union=unary_union, df1=cold_links, df2=temperature, geom1_col='centroid', src_column='REC', axis=1) - del cold_links['geometry'], cold_links['centroid'], temperature['geometry'] + + cold_links.drop(columns=['geometry', 'centroid', 'geometry'], inplace=True) libc.malloc_trim(0) cold_links = cold_links.merge(temperature, left_on='REC', right_on='REC', how='left') - del cold_links['REC'] + cold_links.drop(columns=['REC'], inplace=True) libc.malloc_trim(0) c_expanded = hot_expanded.merge(cold_links, left_on='Link_ID', right_on='Link_ID', how='left') @@ -874,18 +911,18 @@ class TrafficSector(Sector): uni.remove(o) except Exception: error_fleet_code.append(o) - raise IndexError('There are duplicated values for {0} codes in the cold EF files.'.format(error_fleet_code)) + error_exit('There are duplicated values for {0} codes in the cold EF files.'.format(error_fleet_code)) for tstep in range(len(self.date_array)): if 'pm' in self.source_pollutants: cold_df['pm10_{0}'.format(tstep)] = cold_df['pm_{0}'.format(tstep)] cold_df['pm25_{0}'.format(tstep)] = cold_df['pm_{0}'.format(tstep)] - del cold_df['pm_{0}'.format(tstep)] + cold_df.drop(columns=['pm_{0}'.format(tstep)], inplace=True) libc.malloc_trim(0) if 'voc' in self.source_pollutants and 'ch4' in self.source_pollutants: cold_df['nmvoc_{0}'.format(tstep)] = \ cold_df['voc_{0}'.format(tstep)] - cold_df['ch4_{0}'.format(tstep)] - del cold_df['voc_{0}'.format(tstep)] + cold_df.drop(columns=['voc_{0}'.format(tstep)], inplace=True) libc.malloc_trim(0) else: self.logger.write_log("WARNING! nmvoc emissions cannot be estimated because voc or ch4 are not " + @@ -902,21 +939,20 @@ class TrafficSector(Sector): spent_time = timeit.default_timer() columns_to_delete = ['Road_type', 'Fleet_value'] + ['v_{0}'.format(x) for x in range(len(self.date_array))] - for column_name in columns_to_delete: - del expanded[column_name] + expanded.drop(columns=columns_to_delete, inplace=True) for tstep in range(len(self.date_array)): if 'pm' in self.source_pollutants: expanded['pm10_{0}'.format(tstep)] = expanded['pm_{0}'.format(tstep)] expanded['pm25_{0}'.format(tstep)] = expanded['pm_{0}'.format(tstep)] - del expanded['pm_{0}'.format(tstep)] + expanded.drop(columns=['pm_{0}'.format(tstep)], inplace=True) if 'voc' in self.source_pollutants and 'ch4' in self.source_pollutants: expanded['nmvoc_{0}'.format(tstep)] = expanded['voc_{0}'.format(tstep)] - \ expanded['ch4_{0}'.format(tstep)] # For certain vehicles (mostly diesel) and speeds, in urban road CH4 > than VOC according to COPERT V expanded.loc[expanded['nmvoc_{0}'.format(tstep)] < 0, 'nmvoc_{0}'.format(tstep)] = 0 - del expanded['voc_{0}'.format(tstep)] + expanded.drop(columns=['voc_{0}'.format(tstep)], inplace=True) else: self.logger.write_log("nmvoc emissions cannot be estimated because voc or ch4 are not selected in " + "the pollutant list.") @@ -950,7 +986,7 @@ class TrafficSector(Sector): if pollutant == 'pm': df['pm10_{0}'.format(tstep)] = df[p_column] * 0.6 df['pm25_{0}'.format(tstep)] = df[p_column] * 0.42 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -983,7 +1019,7 @@ class TrafficSector(Sector): if pollutant == 'pm': df.loc[:, 'pm10_{0}'.format(tstep)] = df[p_column] * 0.98 df.loc[:, 'pm25_{0}'.format(tstep)] = df[p_column] * 0.39 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -1014,7 +1050,7 @@ class TrafficSector(Sector): if pollutant == 'pm': df.loc[:, 'pm10_{0}'.format(tstep)] = df[p_column] * 0.5 df.loc[:, 'pm25_{0}'.format(tstep)] = df[p_column] * 0.27 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -1043,11 +1079,12 @@ class TrafficSector(Sector): unary_union = p_factor.unary_union road_link_aux['REC'] = road_link_aux.apply(self.nearest, geom_union=unary_union, df1=road_link_aux, df2=p_factor, geom1_col='centroid', src_column='REC', axis=1) - del road_link_aux['centroid'], p_factor['geometry'] + road_link_aux.drop(columns=['centroid'], inplace=True) + p_factor.drop(columns=['geometry'], inplace=True) road_link_aux = road_link_aux.merge(p_factor, left_on='REC', right_on='REC', how='left') - del road_link_aux['REC'] + road_link_aux.drop(columns=['REC'], inplace=True) pollutants = ['pm'] for pollutant in pollutants: @@ -1056,7 +1093,7 @@ class TrafficSector(Sector): if self.resuspension_correction: df = df.merge(road_link_aux, left_on='Link_ID', right_on='Link_ID', how='left') - del df['road_grad'], df['Road_type'], df['Code'] + df.drop(columns=['road_grad', 'Road_type', 'Code'], inplace=True) for tstep in range(len(self.date_array)): p_column = '{0}_{1}'.format(pollutant, tstep) f_column = 'f_{0}'.format(tstep) @@ -1071,7 +1108,7 @@ class TrafficSector(Sector): df['pm10_{0}'.format(tstep)] = df[p_column] # TODO Check fraction of pm2.5 df['pm25_{0}'.format(tstep)] = df[p_column] * 0.5 - del df[p_column] + df.drop(columns=[p_column], inplace=True) # Cleaning df columns_to_delete = ['f_{0}'.format(x) for x in range(len(self.date_array))] + \ @@ -1112,7 +1149,7 @@ class TrafficSector(Sector): # Reads speciation profile speciation = self.read_profiles(speciation) - del speciation['Copert_V_name'] + speciation.drop(columns=['Copert_V_name'], inplace=True) # Transform dataset into timestep rows instead of timestep columns df = self.transform_df(df) @@ -1164,7 +1201,7 @@ class TrafficSector(Sector): else: mol_w = self.molecular_weights[in_p] except KeyError: - raise AttributeError('{0} not found in the molecular weights file.'.format(in_p)) + error_exit('{0} not found in the molecular weights file.'.format(in_p)) # from g/km.h to mol/km.h or g/km.h (aerosols) df_aux[p] = df_aux.loc[:, p] / mol_w @@ -1173,7 +1210,7 @@ class TrafficSector(Sector): df_out_list.append(df_aux[[p] + ['tstep', 'Link_ID']].groupby(['tstep', 'Link_ID']).sum()) del df_aux - del df[in_p] + df.drop(columns=[in_p], inplace=True) df_out = pd.concat(df_out_list, axis=1) @@ -1300,7 +1337,7 @@ class TrafficSector(Sector): link_grid = pd.read_csv(self.link_to_grid_csv) link_grid = link_grid[link_grid['Link_ID'].isin(link_emissions['Link_ID'].values)] - del link_emissions['geometry'] + link_emissions.drop(columns=['geometry'], inplace=True) link_grid = link_grid.merge(link_emissions, left_on='Link_ID', right_on='Link_ID') if 'Unnamed: 0' in link_grid.columns.values: link_grid.drop(columns=['Unnamed: 0'], inplace=True) @@ -1311,8 +1348,7 @@ class TrafficSector(Sector): cols_to_update.remove('FID') for col in cols_to_update: link_grid.loc[:, col] = link_grid[col] * link_grid['length'] - del link_grid['length'] - link_grid.drop(columns=['Link_ID'], inplace=True) + link_grid.drop(columns=['length', 'Link_ID'], inplace=True) link_grid['layer'] = 0 link_grid = link_grid.groupby(['FID', 'layer', 'tstep']).sum() -- GitLab From fd79aa795871e70aa7763eb209860405ab1f17d2 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 14:12:03 +0200 Subject: [PATCH 15/16] - Point sources: checked profile IDs --- conf/hermes.conf | 6 ++-- hermesv3_bu/sectors/point_source_sector.py | 32 +++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 0a9c137..3ef3cab 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__aviation.nc +output_name = HERMESv3__pointsources.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -138,8 +138,8 @@ crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 -point_sources_processors = 0 -traffic_processors = 1 +point_sources_processors = 1 +traffic_processors = 0 traffic_area_processors = 0 diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index e3c2af6..872470c 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -63,13 +63,43 @@ class PointSourceSector(Sector): self.plume_rise = plume_rise self.catalog = self.read_catalog_shapefile(catalog_path, sector_list) - + self.check_catalog() self.catalog_measured = self.read_catalog_for_measured_emissions(catalog_path, sector_list) self.measured_path = measured_emission_path self.plume_rise_pahts = plume_rise_pahts self.logger.write_time_log('PointSourceSector', '__init__', timeit.default_timer() - spent_time) + def check_catalog(self): + # Checking monthly profiles IDs + links_month = set(np.unique(self.catalog['P_month'].dropna().values)) + month = set(self.monthly_profiles.index.values) + month_res = links_month - month + if len(month_res) > 0: + error_exit("The following monthly profile IDs reported in the point sources shapefile do not appear " + + "in the monthly profiles file. {0}".format(month_res)) + # Checking weekly profiles IDs + links_week = set(np.unique(self.catalog['P_week'].dropna().values)) + week = set(self.weekly_profiles.index.values) + week_res = links_week - week + if len(week_res) > 0: + error_exit("The following weekly profile IDs reported in the point sources shapefile do not appear " + + "in the weekly profiles file. {0}".format(week_res)) + # Checking hourly profiles IDs + links_hour = set(np.unique(self.catalog['P_hour'].dropna().values)) + hour = set(self.hourly_profiles.index.values) + hour_res = links_hour - hour + if len(hour_res) > 0: + error_exit("The following hourly profile IDs reported in the point sources shapefile do not appear " + + "in the hourly profiles file. {0}".format(hour_res)) + # Checking specly profiles IDs + links_spec = set(np.unique(self.catalog['P_spec'].dropna().values)) + spec = set(self.speciation_profile.index.values) + spec_res = links_spec - spec + if len(spec_res) > 0: + error_exit("The following speciation profile IDs reported in the point sources shapefile do not appear " + + "in the speciation profiles file. {0}".format(month_res)) + def read_catalog_csv(self, catalog_path, sector_list): """ Read the catalog -- GitLab From 154af87b826d285452f0496ce6f594bd234ceb8c Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 28 Aug 2019 14:46:18 +0200 Subject: [PATCH 16/16] - Point sources: checked columns --- conf/hermes.conf | 6 ++-- hermesv3_bu/sectors/traffic_sector.py | 50 ++++++++++++++++++++------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 3ef3cab..3f70257 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,7 +3,7 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMESv3__pointsources.nc +output_name = HERMESv3__tratra.nc emission_summary = 0 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- @@ -138,8 +138,8 @@ crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 -point_sources_processors = 1 -traffic_processors = 0 +point_sources_processors = 0 +traffic_processors = 1 traffic_area_processors = 0 diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 67a82b3..ee02526 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -344,8 +344,11 @@ class TrafficSector(Sector): if self.comm.Get_rank() == 0: df = gpd.read_file(path) - df.drop(columns=['Adminis', 'CCAA', 'NETWORK_ID', 'Province', 'Road_name', 'aadt_m_sat', 'aadt_m_sun', - 'aadt_m_wd', 'Source'], inplace=True) + try: + df.drop(columns=['Adminis', 'CCAA', 'NETWORK_ID', 'Province', 'Road_name', 'aadt_m_sat', 'aadt_m_sun', + 'aadt_m_wd', 'Source'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) libc.malloc_trim(0) # df.to_file('~/temp/road_links.shp') df = gpd.sjoin(df, self.clip.shapefile.to_crs(df.crs), how="inner", op='intersects') @@ -358,7 +361,11 @@ class TrafficSector(Sector): df = df[df['CONS'] != 0] df = df[df['aadt'] > 0] - df.drop(columns=['CONS'], inplace=True) + try: + df.drop(columns=['CONS'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) + df = df.loc[df['aadt_m_mn'] != 'NULL', :] libc.malloc_trim(0) @@ -429,7 +436,10 @@ class TrafficSector(Sector): # Pollutants different to NH3 if pollutant_name != 'nh3': - df.drop(columns=['Copert_V_name'], inplace=True) + try: + df.drop(columns=['Copert_V_name'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) # For hot emission factors if emission_type == 'hot': @@ -437,8 +447,10 @@ class TrafficSector(Sector): df.loc[df['Technology'].isnull(), 'Technology'] = '' df = df[df['Technology'] != 'EGR'] - - df.drop(columns=['Technology', 'Load'], inplace=True) + try: + df.drop(columns=['Technology', 'Load'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) # Split the EF file into small DataFrames divided by column Road.Slope and Mode restrictions. df_code_slope_road = df[df['Road.Slope'].notnull() & df['Mode'].notnull()] @@ -457,7 +469,10 @@ class TrafficSector(Sector): return df # NH3 pollutant else: - df.drop(columns=['Copert_V_name'], inplace=True) + try: + df.drop(columns=['Copert_V_name'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) # Specific case for cold NH3 emission factors that needs the hot emission factors and the cold ones. if emission_type == 'cold': df_hot = self.read_ef('hot', pollutant_name) @@ -465,7 +480,10 @@ class TrafficSector(Sector): df = df.merge(df_hot, left_on=['Code', 'Mode'], right_on=['Code_hot', 'Mode_hot'], how='left') - df.drop(columns=['Cmileage_hot', 'Mode_hot', 'Code_hot'], inplace=True) + try: + df.drop(columns=['Cmileage_hot', 'Mode_hot', 'Code_hot'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the {0} file'.format(ef_path))) return df @@ -551,7 +569,10 @@ class TrafficSector(Sector): df = df[df['Fleet_value'] > 0] # Deleting unused columns - df.drop(columns=['aadt', 'PcLight', 'PcHeavy', 'PcMoto', 'PcMoped', 'Fleet_Class'], inplace=True) + try: + df.drop(columns=['aadt', 'PcLight', 'PcHeavy', 'PcMoto', 'PcMoped', 'Fleet_Class'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) libc.malloc_trim(0) self.logger.write_time_log('TrafficSector', 'update_fleet_value', timeit.default_timer() - spent_time) @@ -645,10 +666,13 @@ class TrafficSector(Sector): df[['month', 'weekday', 'hour', 'aadt_m_mn', 'aadt_week', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun']]) - df.drop(columns=['month', 'weekday', 'hour', 'P_speed', 'speed_mean', 'sp_wd', 'sp_we', 'sp_hour_mo', - 'sp_hour_tu', 'sp_hour_we', 'sp_hour_th', 'sp_hour_fr', 'sp_hour_sa', 'sp_hour_su', 'aux_date', - 'aadt_m_mn', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun', 'aadt_week', 'start_date'], - inplace=True) + try: + df.drop(columns=['month', 'weekday', 'hour', 'P_speed', 'speed_mean', 'sp_wd', 'sp_we', 'sp_hour_mo', + 'sp_hour_tu', 'sp_hour_we', 'sp_hour_th', 'sp_hour_fr', 'sp_hour_sa', 'sp_hour_su', + 'aux_date', 'aadt_m_mn', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun', 'aadt_week', + 'start_date'], inplace=True) + except KeyError as e: + error_exit(str(e).replace('axis', 'the road links shapefile')) libc.malloc_trim(0) self.logger.write_time_log('TrafficSector', 'calculate_time_dependent_values', -- GitLab