From e3a6a2763b2a4352907cd325c070e0b54a38de44 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 29 Aug 2019 10:14:19 +0200 Subject: [PATCH 1/4] Error exits --- hermesv3_bu/clipping/shapefile_clip.py | 3 ++- hermesv3_bu/config/config.py | 23 +++-------------------- hermesv3_bu/grids/grid.py | 5 +++-- hermesv3_bu/io_server/io_raster.py | 4 ++-- hermesv3_bu/sectors/sector_manager.py | 9 +++++---- hermesv3_bu/writer/cmaq_writer.py | 9 +++++---- hermesv3_bu/writer/monarch_writer.py | 9 +++++---- hermesv3_bu/writer/wrfchem_writer.py | 15 ++++++++------- hermesv3_bu/writer/writer.py | 5 +++-- 9 files changed, 36 insertions(+), 46 deletions(-) diff --git a/hermesv3_bu/clipping/shapefile_clip.py b/hermesv3_bu/clipping/shapefile_clip.py index a0f1ec2..88792ef 100755 --- a/hermesv3_bu/clipping/shapefile_clip.py +++ b/hermesv3_bu/clipping/shapefile_clip.py @@ -6,6 +6,7 @@ import timeit import geopandas as gpd from hermesv3_bu.clipping.clip import Clip from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import error_exit class ShapefileClip(Clip): @@ -48,7 +49,7 @@ class ShapefileClip(Clip): clip = gpd.GeoDataFrame(geometry=[clip.unary_union], crs=clip.crs) clip.to_file(self.shapefile_path) else: - raise IOError(" Clip shapefile {0} not found.") + error_exit(" Clip shapefile {0} not found.") else: clip = gpd.read_file(self.shapefile_path) self.logger.write_log("\tClip created at '{0}'".format(self.shapefile_path), 3) diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 3c9b6bd..26d4ce2 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -1,26 +1,9 @@ #!/usr/bin/env python -# Copyright 2018 Earth Sciences Department, BSC-CNS -# -# This file is part of HERMESv3_BU. -# -# HERMESv3_BU is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# HERMESv3_BU is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with HERMESv3_BU. If not, see . - - from configargparse import ArgParser import os from mpi4py import MPI +from hermesv3_bu.tools.checker import error_exit class Config(ArgParser): @@ -566,10 +549,10 @@ class Config(ArgParser): break except ValueError as e: if str(e) == 'day is out of range for month': - raise ValueError(e) + error_exit(e) if date is None: - raise ValueError("Date format '{0}' not contemplated. Use one of this: {1}".format(str_date, format_types)) + error_exit("Date format '{0}' not contemplated. Use one of this: {1}".format(str_date, format_types)) return date diff --git a/hermesv3_bu/grids/grid.py b/hermesv3_bu/grids/grid.py index 3b85db3..788ec7c 100755 --- a/hermesv3_bu/grids/grid.py +++ b/hermesv3_bu/grids/grid.py @@ -5,6 +5,7 @@ import timeit import numpy as np from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import error_exit def select_grid(comm, logger, arguments): @@ -53,7 +54,7 @@ def select_grid(comm, logger, arguments): arguments.inc_x, arguments.inc_y, arguments.x_0, arguments.y_0) else: - raise NameError('Unknown grid type {0}'.format(arguments.domain_type)) + error_exit('Unknown grid type {0}'.format(arguments.domain_type)) else: grid = None @@ -169,7 +170,7 @@ class Grid(object): else: bound_coords = np.dstack((coords_left, coords_right, coords_right, coords_left)) else: - raise ValueError('ERROR: The number of vertices of the boundaries must be 2 or 4.') + error_exit('The number of vertices of the boundaries must be 2 or 4.') self.logger.write_time_log('Grid', 'create_bounds', timeit.default_timer() - spent_time, 3) return bound_coords diff --git a/hermesv3_bu/io_server/io_raster.py b/hermesv3_bu/io_server/io_raster.py index 9a9d73b..0f6f1a6 100755 --- a/hermesv3_bu/io_server/io_raster.py +++ b/hermesv3_bu/io_server/io_raster.py @@ -10,7 +10,7 @@ import numpy as np from shapely.geometry import Polygon from hermesv3_bu.io_server.io_server import IoServer -from hermesv3_bu.tools.checker import check_files +from hermesv3_bu.tools.checker import check_files, error_exit class IoRaster(IoServer): @@ -169,7 +169,7 @@ class IoRaster(IoServer): else: bound_coords = np.dstack((coords_left, coords_right, coords_right, coords_left)) else: - raise ValueError('ERROR: The number of vertices of the boundaries must be 2 or 4.') + error_exit('ERROR: The number of vertices of the boundaries must be 2 or 4.') # self.logger.write_time_log('IoRaster', 'create_bounds', timeit.default_timer() - spent_time, 3) return bound_coords diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index be6fe1b..f740140 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -2,6 +2,7 @@ import timeit from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import error_exit SECTOR_LIST = ['traffic', 'traffic_area', 'aviation', 'point_sources', 'recreational_boats', 'shipping_port', 'residential', 'livestock', 'crop_operations', 'crop_fertilizers', 'agricultural_machinery'] @@ -224,10 +225,10 @@ class SectorManager(object): sector_dict[sector] = [accum + x for x in range(n_procs)] accum += n_procs if accum != max_procs: - raise ValueError("The selected number of processors '{0}' does not fit ".format(max_procs) + - "with the sum of processors dedicated for all the sectors " + - "'{0}': {1}".format(accum, {sector: len(sector_procs) - for sector, sector_procs in sector_dict.items()})) + error_exit("The selected number of processors '{0}' does not fit ".format(max_procs) + + "with the sum of processors dedicated for all the sectors " + + "'{0}': {1}".format( + accum, {sector: len(sector_procs) for sector, sector_procs in sector_dict.items()})) self.logger.write_time_log('SectorManager', 'make_sector_list', timeit.default_timer() - spent_time) return sector_dict diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 2d12b16..53c5bec 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -9,6 +9,7 @@ from hermesv3_bu.writer.writer import Writer from mpi4py import MPI import timeit from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import error_exit class CmaqWriter(Writer): @@ -67,8 +68,8 @@ class CmaqWriter(Writer): super(CmaqWriter, self).__init__(comm_world, comm_write, logger, netcdf_path, grid, date_array, pollutant_info, rank_distribution, emission_summary) if self.grid.grid_type not in ['Lambert Conformal Conic']: - raise TypeError("ERROR: Only Lambert Conformal Conic grid is implemented for CMAQ. " + - "The current grid type is '{0}'".format(self.grid.grid_type)) + error_exit("Only Lambert Conformal Conic grid is implemented for CMAQ. " + + "The current grid type is '{0}'".format(self.grid.grid_type)) self.global_attributes_order = [ 'IOAPI_VERSION', 'EXEC_ID', 'FTYPE', 'CDATE', 'CTIME', 'WDATE', 'WTIME', 'SDATE', 'STIME', 'TSTEP', 'NTHIK', @@ -111,8 +112,8 @@ class CmaqWriter(Writer): for i, (pollutant, variable) in enumerate(self.pollutant_info.iterrows()): if variable.get('units') not in ['mol.s-1', 'g.s-1', 'mole/s', 'g/s']: - raise ValueError("'{0}' unit is not supported for CMAQ emission ".format(variable.get('units')) + - "input file. Set mol.s-1 or g.s-1 in the speciation_map file.") + error_exit("'{0}' unit is not supported for CMAQ emission ".format(variable.get('units')) + + "input file. Set mol.s-1 or g.s-1 in the speciation_map file.") new_pollutant_info.loc[i, 'pollutant'] = pollutant if variable.get('units') in ['mol.s-1', 'mole/s']: new_pollutant_info.loc[i, 'units'] = "{:<16}".format('mole/s') diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index 20ee47b..7097608 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -6,6 +6,7 @@ from hermesv3_bu.writer.writer import Writer from mpi4py import MPI import timeit from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import error_exit class MonarchWriter(Writer): @@ -62,13 +63,13 @@ class MonarchWriter(Writer): pollutant_info, rank_distribution, emission_summary) if self.grid.grid_type not in ['Rotated']: - raise TypeError("ERROR: Only Rotated grid is implemented for MONARCH. " + - "The current grid type is '{0}'".format(self.grid.grid_type)) + error_exit("ERROR: Only Rotated grid is implemented for MONARCH. " + + "The current grid type is '{0}'".format(self.grid.grid_type)) for i, (pollutant, variable) in enumerate(self.pollutant_info.iterrows()): if variable.get('units') not in ['mol.s-1.m-2', 'kg.s-1.m-2']: - raise ValueError("'{0}' unit is not supported for CMAQ emission ".format(variable.get('units')) + - "input file. Set mol.s-1.m-2 or kg.s-1.m-2 in the speciation_map file.") + error_exit("'{0}' unit is not supported for CMAQ emission ".format(variable.get('units')) + + "input file. Set mol.s-1.m-2 or kg.s-1.m-2 in the speciation_map file.") self.logger.write_time_log('MonarchWriter', '__init__', timeit.default_timer() - spent_time) diff --git a/hermesv3_bu/writer/wrfchem_writer.py b/hermesv3_bu/writer/wrfchem_writer.py index 54d5c05..0f42d9c 100755 --- a/hermesv3_bu/writer/wrfchem_writer.py +++ b/hermesv3_bu/writer/wrfchem_writer.py @@ -9,6 +9,7 @@ from hermesv3_bu.writer.writer import Writer from mpi4py import MPI import timeit from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import error_exit class WrfChemWriter(Writer): @@ -67,8 +68,8 @@ class WrfChemWriter(Writer): super(WrfChemWriter, self).__init__(comm_world, comm_write, logger, netcdf_path, grid, date_array, pollutant_info, rank_distribution, emission_summary) if self.grid.grid_type not in ['Lambert Conformal Conic', 'Mercator']: - raise TypeError("ERROR: Only Lambert Conformal Conic or Mercator grid is implemented for WRF-Chem. " + - "The current grid type is '{0}'".format(self.grid.grid_type)) + error_exit("ERROR: Only Lambert Conformal Conic or Mercator grid is implemented for WRF-Chem. " + + "The current grid type is '{0}'".format(self.grid.grid_type)) self.global_attributes_order = [ 'TITLE', 'START_DATE', 'WEST-EAST_GRID_DIMENSION', 'SOUTH-NORTH_GRID_DIMENSION', @@ -138,9 +139,9 @@ class WrfChemWriter(Writer): for i, (pollutant, variable) in enumerate(self.pollutant_info.iterrows()): if variable.get('units') not in ['mol.h-1.km-2', "mol km^-2 hr^-1", 'ug.s-1.m-2', "ug/m3 m/s"]: - raise ValueError("'{0}' unit is not supported for WRF-Chem emission ".format(variable.get('units')) + - "input file. Set '{0}' in the speciation_map file.".format( - ['mol.h-1.km-2', "mol km^-2 hr^-1", 'ug.s-1.m-2', "ug/m3 m/s"])) + error_exit("'{0}' unit is not supported for WRF-Chem emission ".format(variable.get('units')) + + "input file. Set '{0}' in the speciation_map file.".format( + ['mol.h-1.km-2', "mol km^-2 hr^-1", 'ug.s-1.m-2', "ug/m3 m/s"])) new_pollutant_info.loc[i, 'pollutant'] = pollutant if variable.get('units') in ['mol.h-1.km-2', "mol km^-2 hr^-1"]: @@ -177,8 +178,8 @@ class WrfChemWriter(Writer): elif self.grid.grid_type == 'Mercator': lat_ts = np.float32(self.grid.attributes['lat_ts']) else: - raise TypeError("ERROR: Only Lambert Conformal Conic or Mercator grid is implemented for WRF-Chem. " + - "The current grid type is '{0}'".format(self.grid.grid_type)) + error_exit("Only Lambert Conformal Conic or Mercator grid is implemented for WRF-Chem. " + + "The current grid type is '{0}'".format(self.grid.grid_type)) atts_dict = { 'BOTTOM-TOP_GRID_DIMENSION': np.int32(45), diff --git a/hermesv3_bu/writer/writer.py b/hermesv3_bu/writer/writer.py index f1e9c2c..3d5d284 100755 --- a/hermesv3_bu/writer/writer.py +++ b/hermesv3_bu/writer/writer.py @@ -8,6 +8,7 @@ from mpi4py import MPI from warnings import warn import timeit from hermesv3_bu.logger.log import Log +from hermesv3_bu.tools.checker import error_exit CHUNKING = True BALANCED = False @@ -80,8 +81,8 @@ def select_writer(logger, arguments, grid, date_array): writer = WrfChemWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, rank_distribution, arguments.output_attributes, arguments.emission_summary) else: - raise TypeError("Unknown output model '{0}'. ".format(arguments.output_model) + - "Only MONARCH, CMAQ, WRF_CHEM or DEFAULT writers are available") + error_exit("Unknown output model '{0}'. ".format(arguments.output_model) + + "Only MONARCH, CMAQ, WRF_CHEM or DEFAULT writers are available") logger.write_time_log('Writer', 'select_writer', timeit.default_timer() - spent_time) return writer -- GitLab From fb9619568e734999d1984aa633e5a1f80bcd5515 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 29 Aug 2019 10:47:00 +0200 Subject: [PATCH 2/4] Check netcdf variable names --- conf/hermes.conf | 4 +- hermesv3_bu/io_server/io_netcdf.py | 39 ++++++++--- hermesv3_bu/sectors/point_source_sector.py | 77 +++++++++++++++++----- 3 files changed, 93 insertions(+), 27 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 3f70257..4ab152f 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -138,8 +138,8 @@ crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 residential_processors = 0 recreational_boats_processors = 0 -point_sources_processors = 0 -traffic_processors = 1 +point_sources_processors = 1 +traffic_processors = 0 traffic_area_processors = 0 diff --git a/hermesv3_bu/io_server/io_netcdf.py b/hermesv3_bu/io_server/io_netcdf.py index 6c8480d..a102d83 100755 --- a/hermesv3_bu/io_server/io_netcdf.py +++ b/hermesv3_bu/io_server/io_netcdf.py @@ -12,7 +12,7 @@ from cf_units import num2date, CALENDAR_STANDARD from geopandas import GeoDataFrame from hermesv3_bu.io_server.io_server import IoServer -from hermesv3_bu.tools.checker import check_files +from hermesv3_bu.tools.checker import check_files, error_exit class IoNetcdf(IoServer): @@ -45,11 +45,17 @@ class IoNetcdf(IoServer): """ check_files(netcdf_path) nc = Dataset(netcdf_path, mode='r') - lat_o = nc.variables['latitude'][:] - lon_o = nc.variables['longitude'][:] + try: + lat_o = nc.variables['latitude'][:] + lon_o = nc.variables['longitude'][:] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), netcdf_path)) if date_type == 'daily': - time = nc.variables['time'] + try: + time = nc.variables['time'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), netcdf_path)) # From time array to list of dates. time_array = num2date(time[:], time.units, CALENDAR_STANDARD) time_array = np.array([aux.date() for aux in time_array]) @@ -72,7 +78,10 @@ class IoNetcdf(IoServer): del lat_o, lon_o # Reads the tas variable of the xone and the times needed. - var = nc.variables[var_name][i_time, j_min:j_max, i_min:i_max] + try: + var = nc.variables[var_name][i_time, j_min:j_max, i_min:i_max] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), netcdf_path)) nc.close() var_df = gpd.GeoDataFrame(var.flatten().T, columns=[var_name], crs={'init': 'epsg:4326'}, @@ -110,9 +119,12 @@ class IoNetcdf(IoServer): # self.logger.write_log('Getting temperature from {0}'.format(path), message_level=2) check_files(path) nc = Dataset(path, mode='r') - lat_o = nc.variables['latitude'][:] - lon_o = nc.variables['longitude'][:] - time = nc.variables['time'] + try: + lat_o = nc.variables['latitude'][:] + lon_o = nc.variables['longitude'][:] + time = nc.variables['time'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), path)) # From time array to list of dates. time_array = num2date(time[:], time.units, CALENDAR_STANDARD) i_time = np.where(time_array == date_array[0])[0][0] @@ -134,7 +146,10 @@ class IoNetcdf(IoServer): del lat_o, lon_o # Reads the var variable of the xone and the times needed. - var = nc.variables[var_name][i_time:i_time + (len(date_array)), j_min:j_max, i_min:i_max] + try: + var = nc.variables[var_name][i_time:i_time + (len(date_array)), j_min:j_max, i_min:i_max] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), path)) nc.close() # That condition is fot the cases that the needed temperature is in a different NetCDF. @@ -143,9 +158,13 @@ class IoNetcdf(IoServer): path = os.path.join(netcdf_dir, '{0}_{1}{2}.nc'.format(var_name, aux_date.year, str(aux_date.month).zfill(2))) # self.logger.write_log('Getting {0} from {1}'.format(var_name, path), message_level=2) + check_files(path) nc = Dataset(path, mode='r') i_time = 0 - new_var = nc.variables[var_name][i_time:i_time + (len(date_array) - len(var)), j_min:j_max, i_min:i_max] + try: + new_var = nc.variables[var_name][i_time:i_time + (len(date_array) - len(var)), j_min:j_max, i_min:i_max] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), path)) var = np.concatenate([var, new_var]) diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index 872470c..bb1d2b0 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -437,10 +437,13 @@ class PointSourceSector(Sector): import numpy as np import pandas as pd import geopandas as gpd - + check_files(netcdf_path) nc = Dataset(netcdf_path, mode='r') - lats = nc.variables['lat'][:] - lons = nc.variables['lon'][:] + try: + lats = nc.variables['lat'][:] + lons = nc.variables['lon'][:] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), netcdf_path)) x = np.array([np.arange(lats.shape[1])] * lats.shape[0]) y = np.array([np.arange(lats.shape[0]).T] * lats.shape[1]).T @@ -465,14 +468,21 @@ class PointSourceSector(Sector): from netCDF4 import Dataset, num2date nc_path = os.path.join(dir_path, '{0}_{1}.nc'.format(var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + check_files(nc_path) netcdf = Dataset(nc_path, mode='r') # time_index - time = netcdf.variables['time'] + try: + time = netcdf.variables['time'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), nc_path)) nc_times = [x.replace(minute=0, second=0, microsecond=0) for x in num2date(time[:], time.units, time.calendar)] time_index = nc_times.index(dataframe.name.to_pydatetime().replace(tzinfo=None)) - var = netcdf.variables[var_name][time_index, 0, :] + try: + var = netcdf.variables[var_name][time_index, 0, :] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), nc_path)) netcdf.close() dataframe[var_name] = var[dataframe['Y'], dataframe['X']] @@ -482,14 +492,21 @@ class PointSourceSector(Sector): from netCDF4 import Dataset, num2date nc_path = os.path.join(dir_path, '{0}_{1}.nc'.format(var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + check_files(nc_path) netcdf = Dataset(nc_path, mode='r') # time_index - time = netcdf.variables['time'] + try: + time = netcdf.variables['time'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), nc_path)) nc_times = [x.replace(minute=0, second=0, microsecond=0) for x in num2date(time[:], time.units, time.calendar)] time_index = nc_times.index(dataframe.name.to_pydatetime().replace(tzinfo=None)) - var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) + try: + var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), nc_path)) netcdf.close() var = var[:, dataframe['Y'], dataframe['X']] @@ -513,14 +530,21 @@ class PointSourceSector(Sector): nc_path = os.path.join(dir_path, '{0}_{1}.nc'.format(var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + check_files(nc_path) netcdf = Dataset(nc_path, mode='r') # time_index - time = netcdf.variables['time'] + try: + time = netcdf.variables['time'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), nc_path)) nc_times = [x.replace(minute=0, second=0, microsecond=0) for x in num2date(time[:], time.units, time.calendar)] time_index = nc_times.index(dataframe.name.to_pydatetime().replace(tzinfo=None)) - var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) + try: + var = np.flipud(netcdf.variables[var_name][time_index, :, :, :]) + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), nc_path)) netcdf.close() var = var[:, dataframe['Y'], dataframe['X']] @@ -546,23 +570,34 @@ class PointSourceSector(Sector): # === u10 === u10_nc_path = os.path.join( u_dir_path, '{0}_{1}.nc'.format(u_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + check_files(u10_nc_path) u10_netcdf = Dataset(u10_nc_path, mode='r') # time_index - time = u10_netcdf.variables['time'] + try: + time = u10_netcdf.variables['time'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), u10_nc_path)) nc_times = [x.replace(minute=0, second=0, microsecond=0) for x in num2date(time[:], time.units, time.calendar)] time_index = nc_times.index(dataframe.name.to_pydatetime().replace(tzinfo=None)) - var = u10_netcdf.variables[u_var_name][time_index, 0, :] + try: + var = u10_netcdf.variables[u_var_name][time_index, 0, :] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), u10_nc_path)) u10_netcdf.close() dataframe['u10'] = var[dataframe['Y'], dataframe['X']] # === v10 === v10_nc_path = os.path.join( v_dir_path, '{0}_{1}.nc'.format(v_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + check_files(v10_nc_path) v10_netcdf = Dataset(v10_nc_path, mode='r') - var = v10_netcdf.variables[v_var_name][time_index, 0, :] + try: + var = v10_netcdf.variables[v_var_name][time_index, 0, :] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), v10_nc_path)) v10_netcdf.close() dataframe['v10'] = var[dataframe['Y'], dataframe['X']] @@ -577,14 +612,22 @@ class PointSourceSector(Sector): # === u10 === u10_nc_path = os.path.join( u_dir_path, '{0}_{1}.nc'.format(u_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + check_files(u10_nc_path) u10_netcdf = Dataset(u10_nc_path, mode='r') # time_index - time = u10_netcdf.variables['time'] + try: + time = u10_netcdf.variables['time2'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), u10_nc_path)) nc_times = [x.replace(minute=0, second=0, microsecond=0) for x in num2date(time[:], time.units, time.calendar)] time_index = nc_times.index(dataframe.name.to_pydatetime().replace(tzinfo=None)) - var = np.flipud(u10_netcdf.variables[u_var_name][time_index, :, :, :]) + try: + var = np.flipud(u10_netcdf.variables[u_var_name][time_index, :, :, :]) + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), u10_nc_path)) + u10_netcdf.close() var = var[:, dataframe['Y'], dataframe['X']] @@ -594,9 +637,13 @@ class PointSourceSector(Sector): # === v10 === v10_nc_path = os.path.join( v_dir_path, '{0}_{1}.nc'.format(v_var_name, dataframe.name.replace(hour=0).strftime("%Y%m%d%H"))) + check_files(v10_nc_path) v10_netcdf = Dataset(v10_nc_path, mode='r') - var = np.flipud(v10_netcdf.variables[v_var_name][time_index, :, :, :]) + try: + var = np.flipud(v10_netcdf.variables[v_var_name][time_index, :, :, :]) + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), v10_nc_path)) v10_netcdf.close() var = var[:, dataframe['Y'], dataframe['X']] -- GitLab From a1fff16144782566bc2891b7aff7ad2c7f1a20df Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 29 Aug 2019 10:53:09 +0200 Subject: [PATCH 3/4] PEP8 corrections --- hermesv3_bu/sectors/sector_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index f740140..9f4f5ae 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -228,7 +228,7 @@ class SectorManager(object): error_exit("The selected number of processors '{0}' does not fit ".format(max_procs) + "with the sum of processors dedicated for all the sectors " + "'{0}': {1}".format( - accum, {sector: len(sector_procs) for sector, sector_procs in sector_dict.items()})) + accum, {sector: len(sector_procs) for sector, sector_procs in sector_dict.items()})) self.logger.write_time_log('SectorManager', 'make_sector_list', timeit.default_timer() - spent_time) return sector_dict -- GitLab From bab39a49e6a7150437b2ed62e910c382d0286583 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 29 Aug 2019 10:59:57 +0200 Subject: [PATCH 4/4] -Point source var correction --- hermesv3_bu/sectors/point_source_sector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/sectors/point_source_sector.py b/hermesv3_bu/sectors/point_source_sector.py index bb1d2b0..fd71d8b 100755 --- a/hermesv3_bu/sectors/point_source_sector.py +++ b/hermesv3_bu/sectors/point_source_sector.py @@ -616,7 +616,7 @@ class PointSourceSector(Sector): u10_netcdf = Dataset(u10_nc_path, mode='r') # time_index try: - time = u10_netcdf.variables['time2'] + time = u10_netcdf.variables['time'] except KeyError as e: error_exit("{0} variable not found in {1} file.".format(str(e), u10_nc_path)) nc_times = [x.replace(minute=0, second=0, microsecond=0) for x in -- GitLab