diff --git a/conf/hermes.conf b/conf/hermes.conf index e0be66e13b94182b35d04181877a25cfb99eaba2..8d2ac1969d10fa9b2c788d4becabaed80ebe203f 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -163,16 +163,16 @@ temperature_daily_files_path = /ecmwf/era5/original_files/reorder/dai wind_speed_daily_files_path = /ecmwf/era5/original_files/reorder/daily_mean/sfcWind/ precipitation_files_path = /ecmwf/era5/original_files/reorder/1hourly/prlr/ -temperature_4d_dir = /esarchive/exp/monarch/a1wd/regional/hourly/t -temperature_sfc_dir = /esarchive/exp/monarch/a1wd/regional/hourly/t2 -u_wind_speed_4d_dir = /esarchive/exp/monarch/a1wd/regional/hourly/u -v_wind_speed_4d_dir = /esarchive/exp/monarch/a1wd/regional/hourly/v -u10_wind_speed_dir = /esarchive/exp/monarch/a1wd/regional/hourly/u10 -v10_wind_speed_dir = /esarchive/exp/monarch/a1wd/regional/hourly/v10 -friction_velocity_dir = /esarchive/exp/monarch/a1wd/regional/hourly/ustar -pblh_dir = /esarchive/exp/monarch/a1wd/regional/hourly/mixed_layer_height -obukhov_length_dir = /esarchive/exp/monarch/a1wd/regional/hourly/rmol -layer_thickness_dir = /esarchive/exp/monarch/a1wd/regional/hourly/layer_thickness +temperature_4d_path = /esarchive/exp/monarch/a1wd/regional/hourly/t/t_00.nc +temperature_sfc_path = /esarchive/exp/monarch/a1wd/regional/hourly/t2/t2_00.nc +u_wind_speed_4d_path = /esarchive/exp/monarch/a1wd/regional/hourly/u/u_00.nc +v_wind_speed_4d_path = /esarchive/exp/monarch/a1wd/regional/hourly/v/v_00.nc +u10_wind_speed_path = /esarchive/exp/monarch/a1wd/regional/hourly/u10/u10_00.nc +v10_wind_speed_path = /esarchive/exp/monarch/a1wd/regional/hourly/v10/v10_00.nc +friction_velocity_path = /esarchive/exp/monarch/a1wd/regional/hourly/ustar/ustar_00.nc +pblh_path = /esarchive/exp/monarch/a1wd/regional/hourly/mixed_layer_height/mixed_layer_height_00.nc +obukhov_length_path = /esarchive/exp/monarch/a1wd/regional/hourly/rmol/rmol_00.nc +layer_thickness_path = /esarchive/exp/monarch/a1wd/regional/hourly/layer_thickness/layer_thickness_00.nc [AVIATION SECTOR] diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 2176f95c9db8bf06e52f1e62e5e548e1ef0f15d2..fc1154908a41f12a512ccfa5e8c2973a3b4e6c85 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -234,33 +234,33 @@ class Config(ArgParser): "to make a polygon or nothing to use the default clip: domain extension") # ===== METEO PATHS ===== - p.add_argument('--temperature_hourly_files_path', required=False, type=str, default='True', + p.add_argument('--temperature_hourly_files_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean 2m temperature data.") - p.add_argument('--temperature_daily_files_path', required=False, type=str, default='True', + p.add_argument('--temperature_daily_files_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing daily mean 2m temperature data.") - p.add_argument('--wind_speed_daily_files_path', required=False, type=str, default='True', + p.add_argument('--wind_speed_daily_files_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing daily mean 10m wind speed data.") - p.add_argument('--precipitation_files_path', required=False, type=str, default='True', + p.add_argument('--precipitation_files_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean precipitation data.") - p.add_argument('--temperature_4d_dir', required=False, type=str, default='True', + p.add_argument('--temperature_4d_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean 4D temperature data.") - p.add_argument('--temperature_sfc_dir', required=False, type=str, default='True', + p.add_argument('--temperature_sfc_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean surface temperature data.") - p.add_argument('--u_wind_speed_4d_dir', required=False, type=str, default='True', + p.add_argument('--u_wind_speed_4d_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean 4D U wind component data.") - p.add_argument('--v_wind_speed_4d_dir', required=False, type=str, default='True', + p.add_argument('--v_wind_speed_4d_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean 4D V wind component data.") - p.add_argument('--u10_wind_speed_dir', required=False, type=str, default='True', + p.add_argument('--u10_wind_speed_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing daily mean 10m U wind component data.") - p.add_argument('--v10_wind_speed_dir', required=False, type=str, default='True', + p.add_argument('--v10_wind_speed_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing daily mean 10m V wind component data.") - p.add_argument('--friction_velocity_dir', required=False, type=str, default='True', + p.add_argument('--friction_velocity_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean 4D friction velocity data.") - p.add_argument('--pblh_dir', required=False, type=str, default='True', + p.add_argument('--pblh_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean PBL height data.") - p.add_argument('--obukhov_length_dir', required=False, type=str, default='True', + p.add_argument('--obukhov_length_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean Obukhov length data.") - p.add_argument('--layer_thickness_dir', required=False, type=str, default='True', + p.add_argument('--layer_thickness_path', required=False, type=str, default=None, help="Defines the path to the NetCDF files containing hourly mean 4D layer thickness data.") # ***** AVIATION SECTOR ***** diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index 9a253afe9d58d0d5404eec6df193c0cf90b8d011..edee48a684488969567484cd17ec4886ca0ba570 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -469,10 +469,9 @@ class PointSourceSector(Sector): return dataframe[['X', 'Y']] def get_plumerise_meteo(self, catalog): - def get_sfc_value(dataframe, dir_path, var_name): + def get_sfc_value(dataframe, filepath, var_name): from netCDF4 import Dataset, num2date - nc_path = os.path.join(dir_path, - '{0}_{1}.nc'.format(var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + nc_path = self.parse_netcdf_path(filepath, date_aux=dataframe.name) check_files(nc_path) netcdf = Dataset(nc_path, mode='r') # time_index @@ -493,10 +492,9 @@ class PointSourceSector(Sector): return dataframe[[var_name]] - def get_layers(dataframe, dir_path, var_name): + def get_layers(dataframe, filepath, var_name): from netCDF4 import Dataset, num2date - nc_path = os.path.join(dir_path, - '{0}_{1}.nc'.format(var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + nc_path = self.parse_netcdf_path(filepath, date_aux=dataframe.name) check_files(nc_path) netcdf = Dataset(nc_path, mode='r') # time_index @@ -529,12 +527,11 @@ class PointSourceSector(Sector): return dataframe[['layers']] - def get_temp_top(dataframe, dir_path, var_name): + def get_temp_top(dataframe, filepath, var_name): from netCDF4 import Dataset, num2date from scipy.interpolate import interp1d as interpolate - nc_path = os.path.join(dir_path, - '{0}_{1}.nc'.format(var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + nc_path = self.parse_netcdf_path(filepath, date_aux=dataframe.name) check_files(nc_path) netcdf = Dataset(nc_path, mode='r') # time_index @@ -573,8 +570,7 @@ class PointSourceSector(Sector): def get_wind_speed_10m(dataframe, u_dir_path, v_dir_path, u_var_name, v_var_name): from netCDF4 import Dataset, num2date # === u10 === - u10_nc_path = os.path.join( - u_dir_path, '{0}_{1}.nc'.format(u_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + u10_nc_path = self.parse_netcdf_path(u_dir_path, date_aux=dataframe.name) check_files(u10_nc_path) u10_netcdf = Dataset(u10_nc_path, mode='r') # time_index @@ -594,8 +590,7 @@ class PointSourceSector(Sector): dataframe['u10'] = var[dataframe['Y'], dataframe['X']] # === v10 === - v10_nc_path = os.path.join( - v_dir_path, '{0}_{1}.nc'.format(v_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + v10_nc_path = self.parse_netcdf_path(v_dir_path, date_aux=dataframe.name) check_files(v10_nc_path) v10_netcdf = Dataset(v10_nc_path, mode='r') @@ -615,8 +610,7 @@ class PointSourceSector(Sector): from netCDF4 import Dataset, num2date from scipy.interpolate import interp1d as interpolate # === u10 === - u10_nc_path = os.path.join( - u_dir_path, '{0}_{1}.nc'.format(u_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + u10_nc_path = self.parse_netcdf_path(u_dir_path, date_aux=dataframe.name) check_files(u10_nc_path) u10_netcdf = Dataset(u10_nc_path, mode='r') # time_index @@ -640,8 +634,7 @@ class PointSourceSector(Sector): dataframe['u_{0}'.format(i)] = t_lay # === v10 === - v10_nc_path = os.path.join( - v_dir_path, '{0}_{1}.nc'.format(v_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + v10_nc_path = self.parse_netcdf_path(v_dir_path, date_aux=dataframe.name) check_files(v10_nc_path) v10_netcdf = Dataset(v10_nc_path, mode='r') @@ -670,49 +663,49 @@ class PointSourceSector(Sector): # TODO Use IoNetCDF spent_time = timeit.default_timer() - - meteo_xy = self.get_meteo_xy(catalog.groupby('Code').first(), os.path.join( - self.plume_rise_pahts['temperature_sfc_dir'], - 't2_{0}.nc'.format(self.date_array[0].replace(hour=0).strftime("%Y%m%d%H")))) + print(self.parse_netcdf_path(self.plume_rise_pahts['temperature_sfc_path'], date_aux=self.date_array[0])) + meteo_xy = self.get_meteo_xy(catalog.groupby('Code').first(), + self.parse_netcdf_path(self.plume_rise_pahts['temperature_sfc_path'], + date_aux=self.date_array[0])) catalog = catalog.merge(meteo_xy, left_index=True, right_index=True) # ===== 3D Meteo variables ===== # Adding stc_temp - self.logger.write_log('\t\tGetting temperature from {0}'.format(self.plume_rise_pahts['temperature_sfc_dir']), + self.logger.write_log('\t\tGetting temperature from {0}'.format(self.plume_rise_pahts['temperature_sfc_path']), message_level=3) catalog['temp_sfc'] = catalog.groupby('date_utc')['X', 'Y'].apply( - lambda x: get_sfc_value(x, self.plume_rise_pahts['temperature_sfc_dir'], 't2')) + lambda x: get_sfc_value(x, self.plume_rise_pahts['temperature_sfc_path'], 't2')) self.logger.write_log('\t\tGetting friction velocity from {0}'.format( - self.plume_rise_pahts['friction_velocity_dir']), message_level=3) + self.plume_rise_pahts['friction_velocity_path']), message_level=3) catalog['friction_v'] = catalog.groupby('date_utc')['X', 'Y'].apply( - lambda x: get_sfc_value(x, self.plume_rise_pahts['friction_velocity_dir'], 'ustar')) + lambda x: get_sfc_value(x, self.plume_rise_pahts['friction_velocity_path'], 'ustar')) self.logger.write_log('\t\tGetting PBL height from {0}'.format( - self.plume_rise_pahts['pblh_dir']), message_level=3) + self.plume_rise_pahts['pblh_path']), message_level=3) catalog['pbl'] = catalog.groupby('date_utc')['X', 'Y'].apply( - lambda x: get_sfc_value(x, self.plume_rise_pahts['pblh_dir'], 'mixed_layer_height')) + lambda x: get_sfc_value(x, self.plume_rise_pahts['pblh_path'], 'mixed_layer_height')) self.logger.write_log('\t\tGetting obukhov length from {0}'.format( - self.plume_rise_pahts['obukhov_length_dir']), message_level=3) + self.plume_rise_pahts['obukhov_length_path']), message_level=3) catalog['obukhov_len'] = catalog.groupby('date_utc')['X', 'Y'].apply( - lambda x: get_sfc_value(x, self.plume_rise_pahts['obukhov_length_dir'], 'rmol')) + lambda x: get_sfc_value(x, self.plume_rise_pahts['obukhov_length_path'], 'rmol')) catalog['obukhov_len'] = 1. / catalog['obukhov_len'] self.logger.write_log('\t\tGetting layer thickness from {0}'.format( - self.plume_rise_pahts['layer_thickness_dir']), message_level=3) + self.plume_rise_pahts['layer_thickness_path']), message_level=3) catalog['layers'] = catalog.groupby('date_utc')['X', 'Y'].apply( - lambda x: get_layers(x, self.plume_rise_pahts['layer_thickness_dir'], 'layer_thickness')) + lambda x: get_layers(x, self.plume_rise_pahts['layer_thickness_path'], 'layer_thickness')) self.logger.write_log('\t\tGetting temperatue at the top from {0}'.format( - self.plume_rise_pahts['temperature_4d_dir']), message_level=3) + self.plume_rise_pahts['temperature_4d_path']), message_level=3) catalog['temp_top'] = catalog.groupby('date_utc')['X', 'Y', 'Height', 'layers', 'temp_sfc'].apply( - lambda x: get_temp_top(x, self.plume_rise_pahts['temperature_4d_dir'], 't')) + lambda x: get_temp_top(x, self.plume_rise_pahts['temperature_4d_path'], 't')) self.logger.write_log('\t\tGetting wind speed at 10 m', message_level=3) catalog['wSpeed_10'] = catalog.groupby('date_utc')['X', 'Y'].apply( - lambda x: get_wind_speed_10m(x, self.plume_rise_pahts['u10_wind_speed_dir'], - self.plume_rise_pahts['v10_wind_speed_dir'], 'u10', 'v10')) + lambda x: get_wind_speed_10m(x, self.plume_rise_pahts['u10_wind_speed_path'], + self.plume_rise_pahts['v10_wind_speed_path'], 'u10', 'v10')) self.logger.write_log('\t\tGetting wind speed at the top', message_level=3) catalog['wSpeed_top'] = catalog.groupby('date_utc')['X', 'Y', 'Height', 'layers', 'wSpeed_10'].apply( - lambda x: get_wind_speed_top(x, self.plume_rise_pahts['u_wind_speed_4d_dir'], - self.plume_rise_pahts['v_wind_speed_4d_dir'], 'u', 'v')) + lambda x: get_wind_speed_top(x, self.plume_rise_pahts['u_wind_speed_4d_path'], + self.plume_rise_pahts['v_wind_speed_4d_path'], 'u', 'v')) catalog.drop(columns=['wSpeed_10', 'layers', 'X', 'Y'], inplace=True) self.logger.write_time_log('PointSourceSector', 'get_plumerise_meteo', timeit.default_timer() - spent_time) return catalog diff --git a/hermesv3_bu/sectors/sector.py b/hermesv3_bu/sectors/sector.py index bac090adb382ad76e84f6b7f94ebf6eb29375cfb..73ec9461e906d91efb1cb33925f25b85de3b0180 100755 --- a/hermesv3_bu/sectors/sector.py +++ b/hermesv3_bu/sectors/sector.py @@ -638,3 +638,10 @@ class Sector(object): self.comm.Barrier() return True + + @staticmethod + def parse_netcdf_path(path, date_aux=None): + if date_aux is not None: + path = path.replace('', date_aux.strftime("%Y%m%d%H")) + path = path.replace('', date_aux.strftime("%Y%m%d")) + return path diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index 515823074e6110f0a08b9493474d032a4dbe40e8..6ce18c83f9a751a0cbf5904f8a5c5e8e74a03d5b 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -165,17 +165,17 @@ class SectorManager(object): arguments.point_source_weekly_profiles, arguments.point_source_hourly_profiles, arguments.speciation_map, arguments.point_source_speciation_profiles, arguments.point_source_snaps, arguments.point_source_measured_emissions, arguments.molecular_weights, - plume_rise=arguments.plume_rise, plume_rise_filename=plume_rise_out_file, plume_rise_pahts={ - 'friction_velocity_dir': arguments.friction_velocity_dir, - 'pblh_dir': arguments.pblh_dir, - 'obukhov_length_dir': arguments.obukhov_length_dir, - 'layer_thickness_dir': arguments.layer_thickness_dir, - 'temperature_sfc_dir': arguments.temperature_sfc_dir, - 'temperature_4d_dir': arguments.temperature_4d_dir, - 'u10_wind_speed_dir': arguments.u10_wind_speed_dir, - 'v10_wind_speed_dir': arguments.v10_wind_speed_dir, - 'u_wind_speed_4d_dir': arguments.u_wind_speed_4d_dir, - 'v_wind_speed_4d_dir': arguments.v_wind_speed_4d_dir}) + plume_rise=arguments.plume_rise, plume_rise_pahts={ + 'friction_velocity_path': arguments.friction_velocity_path, + 'pblh_path': arguments.pblh_path, + 'obukhov_length_path': arguments.obukhov_length_path, + 'layer_thickness_path': arguments.layer_thickness_path, + 'temperature_sfc_path': arguments.temperature_sfc_path, + 'temperature_4d_path': arguments.temperature_4d_path, + 'u10_wind_speed_path': arguments.u10_wind_speed_path, + 'v10_wind_speed_path': arguments.v10_wind_speed_path, + 'u_wind_speed_4d_path': arguments.u_wind_speed_4d_path, + 'v_wind_speed_4d_path': arguments.v_wind_speed_4d_path}) elif sector == 'traffic' and comm_world.Get_rank() in sector_procs: from hermesv3_bu.sectors.traffic_sector import TrafficSector self.sector = TrafficSector( diff --git a/hermesv3_bu/tools/checker.py b/hermesv3_bu/tools/checker.py index 69d124a7f5af62044977cb7cc8393bb1e9b654e1..f7dd6b811844260cd33177fd6cb467d6891fb9da 100644 --- a/hermesv3_bu/tools/checker.py +++ b/hermesv3_bu/tools/checker.py @@ -24,6 +24,7 @@ def check_files(file_path_list, warning=False): warn(error_message.replace('ERROR', 'WARNING')) return False else: + # raise FileNotFoundError(error_message) error_exit(error_message) return True