diff --git a/CHANGELOG b/CHANGELOG index 827673ca73d6f98024e33dd6c42499a93d693fa2..a7b6e9c3f1ac43bfc3ce0888d390d9606066a3bc 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +1.0.3 + 2020/07/06 + + - NetCDF read: Added capability to try to read 'lat' instead of 'latitude' and 'lon' instead of 'longitude' + - Corrected bug on NetCDF compression + - Corrected bug on emission summary + - Added CHUNK capability on writer.py + - R-LINE output bug solved + 1.0.2 2020/05/04 diff --git a/hermesv3_bu/__init__.py b/hermesv3_bu/__init__.py index 7863915fa5f8f014c64c8abff41a138e9b3cd4d3..976498ab9cacbd0ed9a538cebd482e5df53f8efb 100755 --- a/hermesv3_bu/__init__.py +++ b/hermesv3_bu/__init__.py @@ -1 +1 @@ -__version__ = "1.0.2" +__version__ = "1.0.3" diff --git a/hermesv3_bu/io_server/io_netcdf.py b/hermesv3_bu/io_server/io_netcdf.py index 56f57e808b92d22002a50c044b32b25965f93c60..caa95582438d5be6dc00864f99cb049ebc67e747 100755 --- a/hermesv3_bu/io_server/io_netcdf.py +++ b/hermesv3_bu/io_server/io_netcdf.py @@ -49,7 +49,11 @@ class IoNetcdf(IoServer): lat_o = nc.variables['latitude'][:] lon_o = nc.variables['longitude'][:] except KeyError as e: - error_exit("{0} variable not found in {1} file.".format(str(e), netcdf_path)) + try: + lat_o = nc.variables['lat'][:] + lon_o = nc.variables['lon'][:] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), netcdf_path)) if date_type == 'daily': try: @@ -125,7 +129,13 @@ class IoNetcdf(IoServer): n_lat = len(lat_o) time = nc.variables['time'] except KeyError as e: - error_exit("{0} variable not found in {1} file.".format(str(e), path)) + try: + lat_o = nc.variables['lat'][:] + lon_o = nc.variables['lon'][:] + n_lat = len(lat_o) + time = nc.variables['time'] + except KeyError as e: + error_exit("{0} variable not found in {1} file.".format(str(e), path)) # From time array to list of dates. time_array = num2date(time[:], time.units, CALENDAR_STANDARD) i_time = np.where(time_array == date_array[0])[0][0] diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 43410c15f5cec4f64ab5494f711346d544379cfd..610518af2ad521746330da0b3f0e674f7d9c0fc1 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -914,7 +914,10 @@ class TrafficSector(Sector): expanded_aux.drop(columns=['A_urban', 'B_urban', 'A_road', 'B_road', 'M'], inplace=True) libc.malloc_trim(0) gc.collect() - expanded_aux.drop(columns=['road_grad', 'FID', 'Fleet_value'], inplace=True) + + expanded_aux.drop(columns=['road_grad', 'Fleet_value'], inplace=True) + if 'FID' in expanded_aux.columns: + expanded_aux.drop(columns=['FID'], inplace=True) expanded_aux.drop(columns=['f_{0}'.format(x) for x in range(len(self.date_array))], inplace=True) libc.malloc_trim(0) @@ -934,7 +937,9 @@ class TrafficSector(Sector): cold_links.drop(columns=['aadt', 'PcHeavy', 'PcMoto', 'PcMoped', 'sp_wd', 'sp_we', 'sp_hour_su', 'sp_hour_mo', 'sp_hour_tu', 'sp_hour_we', 'sp_hour_th', 'sp_hour_fr', 'sp_hour_sa', 'Road_type', 'aadt_m_mn', 'aadt_h_mn', 'aadt_h_wd', 'aadt_h_sat', 'aadt_h_sun', 'aadt_week', - 'fleet_comp', 'road_grad', 'PcLight', 'start_date', 'FID'], inplace=True) + 'fleet_comp', 'road_grad', 'PcLight', 'start_date'], inplace=True) + if 'FID' in cold_links.columns: + cold_links.drop(columns=['FID'], inplace=True) libc.malloc_trim(0) cold_links['centroid'] = cold_links['geometry'].centroid @@ -1490,7 +1495,10 @@ class TrafficSector(Sector): emissions.drop(columns=['geometry'], inplace=True) for poll in emissions.columns.values: - mol_w = self.molecular_weights[self.speciation_map[poll]] + if poll in ['PMC', 'pmc']: + mol_w = 1 + else: + mol_w = self.molecular_weights[self.speciation_map[poll]] # From g/km.h to g/m.s emissions.loc[:, poll] = emissions.loc[:, poll] * mol_w / (1000 * 3600) diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 8d2f0a63550a174d45ba541dc3b7ab67c8588b73..16043bc77334085c8d0cf0293ea69fc9efc8aa15 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -302,8 +302,12 @@ class CmaqWriter(Writer): var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) try: diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 2a4df19f937172c598c3632dd4798a9d41a8bbc9..c791c32f808ebac7891b83e47c9744ced78eda98 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -7,8 +7,6 @@ from mpi4py import MPI import timeit from hermesv3_bu.logger.log import Log -CHUNK = True - class DefaultWriter(Writer): def __init__(self, comm_world, comm_write, logger, netcdf_path, grid, date_array, pollutant_info, @@ -203,8 +201,14 @@ class DefaultWriter(Writer): var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, + chunksizes=self.rank_distribution[0]['shape']) + + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) @@ -218,14 +222,14 @@ class DefaultWriter(Writer): var.units = self.pollutant_info.loc[var_name, 'units'] var.missing_value = -999.0 var.coordinates = 'lat lon' - if self.grid.grid_type == 'Regular Lat-Lon': - var.grid_mapping = 'Latitude_Longitude' - elif self.grid.grid_type == 'Lambert Conformal Conic': - var.grid_mapping = 'Lambert_Conformal' - elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: - var.grid_mapping = 'rotated_pole' - elif self.grid.grid_type == 'Mercator': - var.grid_mapping = 'mercator' + # if self.grid.grid_type == 'Regular Lat-Lon': + # var.grid_mapping = 'Latitude_Longitude' + # elif self.grid.grid_type == 'Lambert Conformal Conic': + # var.grid_mapping = 'Lambert_Conformal' + # elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: + # var.grid_mapping = 'rotated_pole' + # elif self.grid.grid_type == 'Mercator': + # var.grid_mapping = 'mercator' # ========== METADATA ========== self.logger.write_log('\tCreating NetCDF metadata', message_level=2) @@ -238,12 +242,14 @@ class DefaultWriter(Writer): mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif self.grid.grid_type == 'Lambert Conformal Conic': - mapping = netcdf.createVariable('Lambert_Conformal', 'i') - mapping.grid_mapping_name = "lambert_conformal_conic" - mapping.standard_parallel = [self.grid.attributes['lat_1'], self.grid.attributes['lat_2']] - mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] - mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] + # elif self.grid.grid_type == 'Lambert Conformal Conic': + # mapping = netcdf.createVariable('Lambert_Conformal', 'i') + # mapping.grid_mapping_name = "lambert_conformal_conic" + # mapping.standard_parallel = [self.grid.attributes['lat_2'], self.grid.attributes['lat_1']] + # mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] + # mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] + # mapping.false_easting = self.grid.attributes['x_0'] + # mapping.false_northing = self.grid.attributes['y_0'] elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: mapping = netcdf.createVariable('rotated_pole', 'c') diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index f37a1be2160281f61911fef361de598981d5b7d3..0ac0ed0dd49003c24b6670eee47541f2667e4787 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -201,8 +201,13 @@ class MonarchWriter(Writer): var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim, zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim, + zlib=True, complevel=self.compression_level) + else: + var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) try: diff --git a/hermesv3_bu/writer/wrfchem_writer.py b/hermesv3_bu/writer/wrfchem_writer.py index 4108fbfdd896185f1612737b89a69fb634bac3ee..1f14ddead70c38345819c100f6bb8bd59664bebc 100755 --- a/hermesv3_bu/writer/wrfchem_writer.py +++ b/hermesv3_bu/writer/wrfchem_writer.py @@ -355,9 +355,15 @@ class WrfChemWriter(Writer): ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, - ('Time', 'emissions_zdim', 'south_north', 'west_east',)) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable( + var_name, np.float64, ('Time', 'emissions_zdim', 'south_north', 'west_east',), + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, + ('Time', 'emissions_zdim', 'south_north', 'west_east',)) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) diff --git a/hermesv3_bu/writer/writer.py b/hermesv3_bu/writer/writer.py index 55d4f987918d2d8e912582bb77423d7e050a88f5..9e069b5a1c63b4a445f8900690a741e25931771b 100755 --- a/hermesv3_bu/writer/writer.py +++ b/hermesv3_bu/writer/writer.py @@ -10,8 +10,9 @@ import timeit from hermesv3_bu.logger.log import Log from hermesv3_bu.tools.checker import error_exit -CHUNKING = True -BALANCED = False +CHUNKING = False +BALANCED = not CHUNKING + MPI_TAG_CONSTANT = 10**6 BUFFER_SIZE = 2**28 @@ -68,19 +69,21 @@ def select_writer(logger, comm_world, arguments, grid, date_array): if arguments.output_model == 'DEFAULT': from hermesv3_bu.writer.default_writer import DefaultWriter writer = DefaultWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.emission_summary) + rank_distribution, arguments.compression_level, arguments.emission_summary) elif arguments.output_model == 'MONARCH': from hermesv3_bu.writer.monarch_writer import MonarchWriter writer = MonarchWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.emission_summary) + rank_distribution, arguments.compression_level, arguments.emission_summary) elif arguments.output_model == 'CMAQ': from hermesv3_bu.writer.cmaq_writer import CmaqWriter writer = CmaqWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.output_attributes, arguments.emission_summary) + rank_distribution, arguments.output_attributes, arguments.compression_level, + arguments.emission_summary) elif arguments.output_model == 'WRF_CHEM': from hermesv3_bu.writer.wrfchem_writer import WrfChemWriter writer = WrfChemWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.output_attributes, arguments.emission_summary) + rank_distribution, arguments.output_attributes, arguments.compression_level, + arguments.emission_summary) else: error_exit("Unknown output model '{0}'. ".format(arguments.output_model) + "Only MONARCH, CMAQ, WRF_CHEM or DEFAULT writers are available") @@ -272,6 +275,8 @@ class Writer(object): """ spent_time = timeit.default_timer() + self.chunking = CHUNKING + self.comm_world = comm_world self.comm_write = comm_write self.logger = logger