From 392535df6293d5064761134b74dbb335c38a4cfd Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 26 Jul 2019 12:27:30 +0200 Subject: [PATCH] -Writer: option to do CHUNKS --- hermesv3_bu/writer/default_writer.py | 14 +++++++++----- hermesv3_bu/writer/writer.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 8076de5..65c787a 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -6,7 +6,8 @@ from hermesv3_bu.writer.writer import Writer from mpi4py import MPI import timeit from hermesv3_bu.logger.log import Log -import time + +CHUNK = True class DefaultWriter(Writer): @@ -197,13 +198,16 @@ class DefaultWriter(Writer): # emissions.drop(columns=['Unnamed: 0'], inplace=True) for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) - # var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, - # chunksizes=self.rank_distribution[0]['shape']) - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + if CHUNK: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + if self.comm_write.Get_size() > 1: var.set_collective(True) + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, self.rank_distribution[self.comm_write.Get_rank()]['y_min']: self.rank_distribution[self.comm_write.Get_rank()]['y_max'], diff --git a/hermesv3_bu/writer/writer.py b/hermesv3_bu/writer/writer.py index 5b44d02..0da5743 100755 --- a/hermesv3_bu/writer/writer.py +++ b/hermesv3_bu/writer/writer.py @@ -49,10 +49,10 @@ def select_writer(logger, arguments, grid, date_array): if BALANCED: rank_distribution = get_balanced_distribution(logger, arguments.writing_processors, grid.shape) - else: rank_distribution = get_distribution(logger, arguments.writing_processors, grid.shape) + logger.write_log('Rank distribution: {0}'.format(rank_distribution), message_level=3) if comm_world.Get_rank() < arguments.writing_processors: color = 99 else: -- GitLab