From 6ad95b5973eca9bf9f85f805c742d650046eea0d Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Sun, 31 May 2020 12:02:38 +0200 Subject: [PATCH 1/7] Added chunking for DEFAULT writer --- hermesv3_bu/writer/default_writer.py | 12 ++++++++---- hermesv3_bu/writer/writer.py | 5 ++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 2a4df19..06ea862 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -7,8 +7,6 @@ from mpi4py import MPI import timeit from hermesv3_bu.logger.log import Log -CHUNK = True - class DefaultWriter(Writer): def __init__(self, comm_world, comm_write, logger, netcdf_path, grid, date_array, pollutant_info, @@ -203,8 +201,14 @@ class DefaultWriter(Writer): var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if self.chuncking: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, + chunksizes=self.rank_distribution[0]['shape']) + + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + + if self.comm_write.Get_size() > 1 and not self.chuncking: var.set_collective(True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) diff --git a/hermesv3_bu/writer/writer.py b/hermesv3_bu/writer/writer.py index 55d4f98..cdfb209 100755 --- a/hermesv3_bu/writer/writer.py +++ b/hermesv3_bu/writer/writer.py @@ -11,7 +11,8 @@ from hermesv3_bu.logger.log import Log from hermesv3_bu.tools.checker import error_exit CHUNKING = True -BALANCED = False +BALANCED = not CHUNKING + MPI_TAG_CONSTANT = 10**6 BUFFER_SIZE = 2**28 @@ -272,6 +273,8 @@ class Writer(object): """ spent_time = timeit.default_timer() + self.chunking = CHUNKING + self.comm_world = comm_world self.comm_write = comm_write self.logger = logger -- GitLab From 7450c04cb14e31533e0ee873a1b133ca590f6d9d Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Sun, 31 May 2020 13:13:11 +0200 Subject: [PATCH 2/7] Added chunking for DEFAULT writer --- hermesv3_bu/writer/default_writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 06ea862..7ad003c 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -201,14 +201,14 @@ class DefaultWriter(Writer): var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True, complevel=self.compression_level) else: - if self.chuncking: + if self.chunking: var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, chunksizes=self.rank_distribution[0]['shape']) else: var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1 and not self.chuncking: + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) -- GitLab From 575bfe68bfd6d7fad9892de2fd7f11db63731230 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Sun, 31 May 2020 18:31:19 +0200 Subject: [PATCH 3/7] Corrected bug on NetCDF compression --- hermesv3_bu/writer/writer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hermesv3_bu/writer/writer.py b/hermesv3_bu/writer/writer.py index cdfb209..9e069b5 100755 --- a/hermesv3_bu/writer/writer.py +++ b/hermesv3_bu/writer/writer.py @@ -10,7 +10,7 @@ import timeit from hermesv3_bu.logger.log import Log from hermesv3_bu.tools.checker import error_exit -CHUNKING = True +CHUNKING = False BALANCED = not CHUNKING MPI_TAG_CONSTANT = 10**6 @@ -69,19 +69,21 @@ def select_writer(logger, comm_world, arguments, grid, date_array): if arguments.output_model == 'DEFAULT': from hermesv3_bu.writer.default_writer import DefaultWriter writer = DefaultWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.emission_summary) + rank_distribution, arguments.compression_level, arguments.emission_summary) elif arguments.output_model == 'MONARCH': from hermesv3_bu.writer.monarch_writer import MonarchWriter writer = MonarchWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.emission_summary) + rank_distribution, arguments.compression_level, arguments.emission_summary) elif arguments.output_model == 'CMAQ': from hermesv3_bu.writer.cmaq_writer import CmaqWriter writer = CmaqWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.output_attributes, arguments.emission_summary) + rank_distribution, arguments.output_attributes, arguments.compression_level, + arguments.emission_summary) elif arguments.output_model == 'WRF_CHEM': from hermesv3_bu.writer.wrfchem_writer import WrfChemWriter writer = WrfChemWriter(comm_world, comm_write, logger, arguments.output_name, grid, date_array, pollutant_info, - rank_distribution, arguments.output_attributes, arguments.emission_summary) + rank_distribution, arguments.output_attributes, arguments.compression_level, + arguments.emission_summary) else: error_exit("Unknown output model '{0}'. ".format(arguments.output_model) + "Only MONARCH, CMAQ, WRF_CHEM or DEFAULT writers are available") -- GitLab From 3961dced69f0790b6c72875ba778b429cb8c9991 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Sun, 31 May 2020 18:48:03 +0200 Subject: [PATCH 4/7] Added chunking for all the writer types --- hermesv3_bu/writer/cmaq_writer.py | 8 ++++++-- hermesv3_bu/writer/monarch_writer.py | 9 +++++++-- hermesv3_bu/writer/wrfchem_writer.py | 12 +++++++++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 8d2f0a6..16043bc 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -302,8 +302,12 @@ class CmaqWriter(Writer): var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) try: diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index f37a1be..0ac0ed0 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -201,8 +201,13 @@ class MonarchWriter(Writer): var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim, zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim, + zlib=True, complevel=self.compression_level) + else: + var = netcdf.createVariable(var_name, 'f', ('time', 'lev',) + var_dim) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) try: diff --git a/hermesv3_bu/writer/wrfchem_writer.py b/hermesv3_bu/writer/wrfchem_writer.py index 4108fbf..1f14dde 100755 --- a/hermesv3_bu/writer/wrfchem_writer.py +++ b/hermesv3_bu/writer/wrfchem_writer.py @@ -355,9 +355,15 @@ class WrfChemWriter(Writer): ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=True, complevel=self.compression_level) else: - var = netcdf.createVariable(var_name, np.float64, - ('Time', 'emissions_zdim', 'south_north', 'west_east',)) - if self.comm_write.Get_size() > 1: + if self.chunking: + var = netcdf.createVariable( + var_name, np.float64, ('Time', 'emissions_zdim', 'south_north', 'west_east',), + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, + ('Time', 'emissions_zdim', 'south_north', 'west_east',)) + + if self.comm_write.Get_size() > 1 and not self.chunking: var.set_collective(True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) -- GitLab From d2915ff30dd635fe151f218109d6cc69db82b709 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 2 Jun 2020 12:50:35 +0200 Subject: [PATCH 5/7] Correcting Error on NetCDF LCC definition --- hermesv3_bu/writer/default_writer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 7ad003c..932bd00 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -248,6 +248,8 @@ class DefaultWriter(Writer): mapping.standard_parallel = [self.grid.attributes['lat_1'], self.grid.attributes['lat_2']] mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] + mapping.false_easting = self.grid.attributes['x_0'] + mapping.false_northing = self.grid.attributes['y_0'] elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: mapping = netcdf.createVariable('rotated_pole', 'c') -- GitLab From 7aac511b6c7c0ada604f0dc979ce651565939642 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 2 Jun 2020 14:01:02 +0200 Subject: [PATCH 6/7] Correcting Error on NetCDF grid mapping definition --- hermesv3_bu/writer/default_writer.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 932bd00..86f80f1 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -222,14 +222,14 @@ class DefaultWriter(Writer): var.units = self.pollutant_info.loc[var_name, 'units'] var.missing_value = -999.0 var.coordinates = 'lat lon' - if self.grid.grid_type == 'Regular Lat-Lon': - var.grid_mapping = 'Latitude_Longitude' - elif self.grid.grid_type == 'Lambert Conformal Conic': - var.grid_mapping = 'Lambert_Conformal' - elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: - var.grid_mapping = 'rotated_pole' - elif self.grid.grid_type == 'Mercator': - var.grid_mapping = 'mercator' + # if self.grid.grid_type == 'Regular Lat-Lon': + # var.grid_mapping = 'Latitude_Longitude' + # elif self.grid.grid_type == 'Lambert Conformal Conic': + # var.grid_mapping = 'Lambert_Conformal' + # elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: + # var.grid_mapping = 'rotated_pole' + # elif self.grid.grid_type == 'Mercator': + # var.grid_mapping = 'mercator' # ========== METADATA ========== self.logger.write_log('\tCreating NetCDF metadata', message_level=2) @@ -245,7 +245,7 @@ class DefaultWriter(Writer): elif self.grid.grid_type == 'Lambert Conformal Conic': mapping = netcdf.createVariable('Lambert_Conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" - mapping.standard_parallel = [self.grid.attributes['lat_1'], self.grid.attributes['lat_2']] + mapping.standard_parallel = [self.grid.attributes['lat_2'], self.grid.attributes['lat_1']] mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] mapping.false_easting = self.grid.attributes['x_0'] -- GitLab From f02926503f2af25b5e8db804d4112b033bdec5ad Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 2 Jun 2020 14:41:34 +0200 Subject: [PATCH 7/7] Correcting Error on NetCDF grid mapping definition --- hermesv3_bu/writer/default_writer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 86f80f1..c791c32 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -242,14 +242,14 @@ class DefaultWriter(Writer): mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif self.grid.grid_type == 'Lambert Conformal Conic': - mapping = netcdf.createVariable('Lambert_Conformal', 'i') - mapping.grid_mapping_name = "lambert_conformal_conic" - mapping.standard_parallel = [self.grid.attributes['lat_2'], self.grid.attributes['lat_1']] - mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] - mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] - mapping.false_easting = self.grid.attributes['x_0'] - mapping.false_northing = self.grid.attributes['y_0'] + # elif self.grid.grid_type == 'Lambert Conformal Conic': + # mapping = netcdf.createVariable('Lambert_Conformal', 'i') + # mapping.grid_mapping_name = "lambert_conformal_conic" + # mapping.standard_parallel = [self.grid.attributes['lat_2'], self.grid.attributes['lat_1']] + # mapping.longitude_of_central_meridian = self.grid.attributes['lon_0'] + # mapping.latitude_of_projection_origin = self.grid.attributes['lat_0'] + # mapping.false_easting = self.grid.attributes['x_0'] + # mapping.false_northing = self.grid.attributes['y_0'] elif self.grid.grid_type in ['Rotated', 'Rotated_nested']: mapping = netcdf.createVariable('rotated_pole', 'c') -- GitLab