diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 8d2f0a63550a174d45ba541dc3b7ab67c8588b73..16043bc77334085c8d0cf0293ea69fc9efc8aa15 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -302,8 +302,12 @@ class CmaqWriter(Writer): var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) try: diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 2a4df19f937172c598c3632dd4798a9d41a8bbc9..c791c32f808ebac7891b83e47c9744ced78eda98 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -7,8 +7,6 @@ from mpi4py import MPI import timeit from hermesv3_bu.logger.log import Log -CHUNK = True - class DefaultWriter(Writer): def __init__(self, comm_world, comm_write, logger, netcdf_path, grid, date_array, pollutant_info, @@ -203,8 +201,14 @@ class DefaultWriter(Writer): var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, + chunksizes=self.rank_distribution[0]['shape']) + + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) @@ -218,14 +222,14 @@ class DefaultWriter(Writer): var.units = self.pollutant_info.loc[var_name, 'units'] var.missing_value = -999.0 var.coordinates = 'lat lon' - if self.grid.grid_type == 'Regular Lat-Lon': - var.grid_mapping = 'Latitude_Longitude' - elif self.grid.grid_type == 'Lambert Conformal Conic': - var.grid_mapping = 'Lambert_Conformal' - elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: - var.grid_mapping = 'rotated_pole' - elif self.grid.grid_type == 'Mercator': - var.grid_mapping = 'mercator' + # if self.grid.grid_type == 'Regular Lat-Lon': + # var.grid_mapping = 'Latitude_Longitude' + # elif self.grid.grid_type == 'Lambert Conformal Conic': + # var.grid_mapping = 'Lambert_Conformal' + # elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: + # var.grid_mapping = 'rotated_pole' + # elif self.grid.grid_type == 'Mercator': + # var.grid_mapping = 'mercator' # ========== METADATA ========== self.logger.write_log('\tCreating NetCDF metadata', message_level=2) @@ -238,12 +242,14 @@ class DefaultWriter(Writer): mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif self.grid.grid_type == 'Lambert Conformal Conic': - mapping = netcdf.createVariable('Lambert_Conformal', 'i') - mapping.grid_mapping_name = "lambert_conformal_conic" - mapping.standard_parallel = [self.grid.attributes['lat_1'], self.grid.attributes['lat_2']] - mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] - mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] + # elif self.grid.grid_type == 'Lambert Conformal Conic': + # mapping = netcdf.createVariable('Lambert_Conformal', 'i') + # mapping.grid_mapping_name = "lambert_conformal_conic" + # mapping.standard_parallel = [self.grid.attributes['lat_2'], self.grid.attributes['lat_1']] + # mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] + # mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] + # mapping.false_easting = self.grid.attributes['x_0'] + # mapping.false_northing = self.grid.attributes['y_0'] elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: mapping = netcdf.createVariable('rotated_pole', 'c') diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index f37a1be2160281f61911fef361de598981d5b7d3..0ac0ed0dd49003c24b6670eee47541f2667e4787 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -201,8 +201,13 @@ class MonarchWriter(Writer): var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim, zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim, + zlib=True, complevel=self.compression_level) + else: + var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) try: diff --git a/hermesv3_bu/writer/wrfchem_writer.py b/hermesv3_bu/writer/wrfchem_writer.py index 4108fbfdd896185f1612737b89a69fb634bac3ee..1f14ddead70c38345819c100f6bb8bd59664bebc 100755 --- a/hermesv3_bu/writer/wrfchem_writer.py +++ b/hermesv3_bu/writer/wrfchem_writer.py @@ -355,9 +355,15 @@ class WrfChemWriter(Writer): ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, - ('Time', 'emissions_zdim', 'south_north', 'west_east',)) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable( + var_name, np.float64, ('Time', 'emissions_zdim', 'south_north', 'west_east',), + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, + ('Time', 'emissions_zdim', 'south_north', 'west_east',)) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) diff --git a/hermesv3_bu/writer/writer.py b/hermesv3_bu/writer/writer.py index 55d4f987918d2d8e912582bb77423d7e050a88f5..9e069b5a1c63b4a445f8900690a741e25931771b 100755 --- a/hermesv3_bu/writer/writer.py +++ b/hermesv3_bu/writer/writer.py @@ -10,8 +10,9 @@ import timeit from hermesv3_bu.logger.log import Log from hermesv3_bu.tools.checker import error_exit -CHUNKING = True -BALANCED = False +CHUNKING = False +BALANCED = not CHUNKING + MPI_TAG_CONSTANT = 10**6 BUFFER_SIZE = 2**28 @@ -68,19 +69,21 @@ def select_writer(logger, comm_world, arguments, grid, date_array): if arguments.output_model == 'DEFAULT': from hermesv3_bu.writer.default_writer import DefaultWriter writer = DefaultWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.emission_summary) + rank_distribution, arguments.compression_level, arguments.emission_summary) elif arguments.output_model == 'MONARCH': from hermesv3_bu.writer.monarch_writer import MonarchWriter writer = MonarchWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.emission_summary) + rank_distribution, arguments.compression_level, arguments.emission_summary) elif arguments.output_model == 'CMAQ': from hermesv3_bu.writer.cmaq_writer import CmaqWriter writer = CmaqWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.output_attributes, arguments.emission_summary) + rank_distribution, arguments.output_attributes, arguments.compression_level, + arguments.emission_summary) elif arguments.output_model == 'WRF_CHEM': from hermesv3_bu.writer.wrfchem_writer import WrfChemWriter writer = WrfChemWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.output_attributes, arguments.emission_summary) + rank_distribution, arguments.output_attributes, arguments.compression_level, + arguments.emission_summary) else: error_exit("Unknown output model '{0}'. ".format(arguments.output_model) + "Only MONARCH, CMAQ, WRF_CHEM or DEFAULT writers are available") @@ -272,6 +275,8 @@ class Writer(object): """ spent_time = timeit.default_timer() + self.chunking = CHUNKING + self.comm_world = comm_world self.comm_write = comm_write self.logger = logger