From 04b6d943c31dc84ff066de2bd77009c0dad0c40f Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 19 Aug 2019 16:33:28 +0200 Subject: [PATCH 01/11] Added serial option to all writers --- hermesv3_bu/writer/cmaq_writer.py | 20 ++++++++++++++------ hermesv3_bu/writer/monarch_writer.py | 12 ++++++++++-- hermesv3_bu/writer/wrfchem_writer.py | 13 +++++++++++-- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 0649c89..18d67e7 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -268,7 +268,11 @@ class CmaqWriter(Writer): """ spent_time = timeit.default_timer() - netcdf = Dataset(self.netcdf_path, mode='w', parallel=True, comm=self.comm_write, info=MPI.Info()) + if self.comm_write.Get_size() > 1: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w', parallel=True, comm=self.comm_write, + info=MPI.Info()) + else: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w') # ===== DIMENSIONS ===== self.logger.write_log('\tCreating NetCDF dimensions', message_level=2) @@ -291,13 +295,17 @@ class CmaqWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) + if self.comm_write.Get_size() > 1: + var.set_collective(True) + + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) + var[:, :, - self.rank_distribution[self.comm_write.Get_rank()]['y_min']: - self.rank_distribution[self.comm_write.Get_rank()]['y_max'], - self.rank_distribution[self.comm_write.Get_rank()]['x_min']: - self.rank_distribution[self.comm_write.Get_rank()]['x_max']] = var_data + self.rank_distribution[self.comm_write.Get_rank()]['y_min']: + self.rank_distribution[self.comm_write.Get_rank()]['y_max'], + self.rank_distribution[self.comm_write.Get_rank()]['x_min']: + self.rank_distribution[self.comm_write.Get_rank()]['x_max']] = var_data var.long_name = self.pollutant_info.loc[var_name, 'long_name'] var.units = self.pollutant_info.loc[var_name, 'units'] diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index 7a00324..8981d51 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -113,7 +113,11 @@ class MonarchWriter(Writer): """ from cf_units import Unit spent_time = timeit.default_timer() - netcdf = Dataset(self.netcdf_path, mode='w', parallel=True, comm=self.comm_write, info=MPI.Info()) + if self.comm_write.Get_size() > 1: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w', parallel=True, comm=self.comm_write, + info=MPI.Info()) + else: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w') # ========== DIMENSIONS ========== self.logger.write_log('\tCreating NetCDF dimensions', message_level=2) @@ -186,11 +190,15 @@ class MonarchWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) # var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, # chunksizes=self.rank_distribution[0]['shape']) var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + if self.comm_write.Get_size() > 1: + var.set_collective(True) + + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) + var[:, :, self.rank_distribution[self.comm_write.Get_rank()]['y_min']: self.rank_distribution[self.comm_write.Get_rank()]['y_max'], diff --git a/hermesv3_bu/writer/wrfchem_writer.py b/hermesv3_bu/writer/wrfchem_writer.py index 4943f99..2d24d01 100755 --- a/hermesv3_bu/writer/wrfchem_writer.py +++ b/hermesv3_bu/writer/wrfchem_writer.py @@ -327,7 +327,11 @@ class WrfChemWriter(Writer): """ spent_time = timeit.default_timer() - netcdf = Dataset(self.netcdf_path, mode='w', parallel=True, comm=self.comm_write, info=MPI.Info()) + if self.comm_write.Get_size() > 1: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w', parallel=True, comm=self.comm_write, + info=MPI.Info()) + else: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w') # ===== DIMENSIONS ===== self.logger.write_log('\tCreating NetCDF dimensions', message_level=2) @@ -347,8 +351,13 @@ class WrfChemWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var = netcdf.createVariable(var_name, np.float64, ('Time', 'emissions_zdim', 'south_north', 'west_east',)) + + if self.comm_write.Get_size() > 1: + var.set_collective(True) + + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) + var[:, :, self.rank_distribution[self.comm_write.Get_rank()]['y_min']: self.rank_distribution[self.comm_write.Get_rank()]['y_max'], -- GitLab From 707c616b71a2510bf5c8e0dda01182c08293f9b5 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 19 Aug 2019 17:17:22 +0200 Subject: [PATCH 02/11] popullation to population --- conf/hermes.conf | 2 +- hermesv3_bu/config/config.py | 2 +- hermesv3_bu/sectors/sector_manager.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 6088cfe..6809617 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -322,7 +322,7 @@ traffic_speciation_profile_resuspension = /profiles/speciation/traffi traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 do_evaporative = 1 traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csv -popullation_by_municipality = /traffic_area/population_by_mun.csv +population_by_municipality = /traffic_area/population_by_mun.csv traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_cmaq.csv traffic_area_evaporative_ef_file = /traffic_area/ef/evaporative_nmvoc.csv do_small_cities = 1 diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 3eab46d..5ed5bd7 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -353,7 +353,7 @@ class Config(ArgParser): p.add_argument('--traffic_area_pollutants', required=False, help='...') p.add_argument('--do_evaporative', required=False, help='...') p.add_argument('--traffic_area_gas_path', required=False, help='...') - p.add_argument('--popullation_by_municipality', required=False, help='...') + p.add_argument('--population_by_municipality', required=False, help='...') p.add_argument('--traffic_area_speciation_profiles_evaporative', required=False, help='...') p.add_argument('--traffic_area_evaporative_ef_file', required=False, help='...') p.add_argument('--do_small_cities', required=False, help='...') diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index 51b355d..b306dca 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -196,7 +196,7 @@ class SectorManager(object): arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.traffic_area_pollutants, grid.vertical_desctiption, arguments.population_density_map, arguments.speciation_map, arguments.molecular_weights, arguments.do_evaporative, arguments.traffic_area_gas_path, - arguments.popullation_by_municipality, arguments.nut_shapefile_prov, + arguments.population_by_municipality, arguments.nut_shapefile_prov, arguments.traffic_area_speciation_profiles_evaporative, arguments.traffic_area_evaporative_ef_file, arguments.temperature_hourly_files_path, arguments.do_small_cities, arguments.traffic_area_small_cities_path, arguments.traffic_area_speciation_profiles_small_cities, -- GitLab From 6d4956239f50a43a5cd20696f2612e31b8e6c95c Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 19 Aug 2019 17:37:55 +0200 Subject: [PATCH 03/11] PEP8 corrections --- hermesv3_bu/writer/cmaq_writer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 18d67e7..4670e6f 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -302,10 +302,10 @@ class CmaqWriter(Writer): var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, - self.rank_distribution[self.comm_write.Get_rank()]['y_min']: - self.rank_distribution[self.comm_write.Get_rank()]['y_max'], - self.rank_distribution[self.comm_write.Get_rank()]['x_min']: - self.rank_distribution[self.comm_write.Get_rank()]['x_max']] = var_data + self.rank_distribution[self.comm_write.Get_rank()]['y_min']: + self.rank_distribution[self.comm_write.Get_rank()]['y_max'], + self.rank_distribution[self.comm_write.Get_rank()]['x_min']: + self.rank_distribution[self.comm_write.Get_rank()]['x_max']] = var_data var.long_name = self.pollutant_info.loc[var_name, 'long_name'] var.units = self.pollutant_info.loc[var_name, 'units'] -- GitLab From f0ebc50269724cf7b083d2b8f4dab0e39ced9732 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 19 Aug 2019 17:46:48 +0200 Subject: [PATCH 04/11] Fixed imports --- hermesv3_bu/grids/grid_latlon.py | 18 ------------------ hermesv3_bu/grids/grid_mercator.py | 2 +- hermesv3_bu/grids/grid_rotated.py | 2 +- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/hermesv3_bu/grids/grid_latlon.py b/hermesv3_bu/grids/grid_latlon.py index 8e1cd95..d1c0513 100755 --- a/hermesv3_bu/grids/grid_latlon.py +++ b/hermesv3_bu/grids/grid_latlon.py @@ -1,23 +1,5 @@ #!/usr/bin/env python -# Copyright 2018 Earth Sciences Department, BSC-CNS -# -# This file is part of HERMESv3_GR. -# -# HERMESv3_GR is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# HERMESv3_GR is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with HERMESv3_GR. If not, see . - - import os import timeit diff --git a/hermesv3_bu/grids/grid_mercator.py b/hermesv3_bu/grids/grid_mercator.py index f03ccd7..24faf79 100755 --- a/hermesv3_bu/grids/grid_mercator.py +++ b/hermesv3_bu/grids/grid_mercator.py @@ -4,7 +4,7 @@ import os import timeit import numpy as np from pyproj import Proj -from grid import Grid +from hermesv3_bu.grids.grid import Grid from hermesv3_bu.logger.log import Log diff --git a/hermesv3_bu/grids/grid_rotated.py b/hermesv3_bu/grids/grid_rotated.py index f8b1a1d..02237f9 100755 --- a/hermesv3_bu/grids/grid_rotated.py +++ b/hermesv3_bu/grids/grid_rotated.py @@ -2,7 +2,7 @@ import os import timeit -from grid import Grid +from hermesv3_bu.grids.grid import Grid import numpy as np import math -- GitLab From 27f6f306cf0d471038808ebbe0ff711250ef9863 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 20 Aug 2019 14:13:17 +0200 Subject: [PATCH 05/11] Corrected bug on calculating Fleet value for each vehicle type --- conf/hermes.conf | 43 +++++++++++++++++++-------- hermesv3_bu/sectors/traffic_sector.py | 11 +++---- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 6809617..f3df917 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,20 +3,20 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMES_.nc +output_name = HERMESv3__rot.nc emission_summary = 1 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 output_timestep_num = 25 -auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/py3/_ +auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ erase_auxiliary_files = 0 [DOMAIN] # domain_type=[lcc, rotated, mercator, regular] -domain_type = lcc +domain_type = rotated # output_type=[MONARCH, CMAQ, WRF_CHEM, DEFAULT] output_model = DEFAULT output_attributes = /writing/global_attributes_WRF-Chem.csv @@ -24,12 +24,19 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver #vertical_description = /profiles/vertical/CMAQ_15layers_vertical_description.csv # if domain_type == rotated: - centre_lat = 51 - centre_lon = 10 - west_boundary = -35 - south_boundary = -27 - inc_rlat = 0.2 - inc_rlon = 0.2 + # centre_lat = 51 + # centre_lon = 10 + # west_boundary = -35 + # south_boundary = -27 + # inc_rlat = 0.2 + # inc_rlon = 0.2 + + centre_lat = 40.5 + centre_lon = -3.5 + west_boundary = -7.0 + south_boundary = -7.0 + inc_rlat = 0.4 + inc_rlon = 0.4 # if domain_type == lcc: @@ -117,16 +124,28 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver [SECTOR MANAGEMENT] writing_processors = 12 +# aviation_processors = 1 +# shipping_port_processors = 1 +# livestock_processors = 12 +# crop_operations_processors = 1 +# crop_fertilizers_processors = 4 +# agricultural_machinery_processors = 1 +# residential_processors = 4 +# recreational_boats_processors = 4 +# point_sources_processors = 16 +# traffic_processors = 256 +# traffic_area_processors = 1 + aviation_processors = 0 shipping_port_processors = 0 livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 -residential_processors = 1 +residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 -traffic_processors = 0 +traffic_processors = 1 traffic_area_processors = 0 @@ -299,7 +318,7 @@ resuspension_correction = 1 write_rline = 0 traffic_pollutants = nox_no2, nh3, co, so2, pm, voc, ch4 -vehicle_types = PCD_13 PCD_14 PCD_15 PCG_25 PCG_26 PCG_27 +# vehicle_types = PCD_13 PCD_14 PCD_15 PCG_25 PCG_26 PCG_27 # load = [0, 0.5, 1] load = 0.5 road_link_path = /traffic/road_links/2015/road_links_2015.shp diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 72f2acd..27636b2 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -255,8 +255,10 @@ class TrafficSector(Sector): df.drop(columns=['Adminis', 'CCAA', 'NETWORK_ID', 'Province', 'Road_name', 'aadt_m_sat', 'aadt_m_sun', 'aadt_m_wd', 'Source'], inplace=True) libc.malloc_trim(0) + # df.to_file('~/temp/road_links.shp') df = gpd.sjoin(df, self.clip.shapefile.to_crs(df.crs), how="inner", op='intersects') - df.drop(columns=['index_right', 'FID'], inplace=True) + # df.to_file('~/temp/road_links_selected.shp') + df.drop(columns=['index_right'], inplace=True) libc.malloc_trim(0) # Filtering road links to CONSiderate. @@ -436,17 +438,16 @@ class TrafficSector(Sector): if x.name == 'light_veh': x['value'] = x['PcLight'].mul(x['Fleet_value'] * x['aadt'], axis='index') elif x.name == 'heavy_veh': - x['value'] = x['PcHeavy'].mul(x['Fleet_value'], axis='index') + x['value'] = x['PcHeavy'].mul(x['Fleet_value'] * x['aadt'], axis='index') elif x.name == 'motos': - x['value'] = x['PcMoto'].mul(x['Fleet_value'], axis='index') + x['value'] = x['PcMoto'].mul(x['Fleet_value'] * x['aadt'], axis='index') elif x.name == 'mopeds': - x['value'] = x['PcMoped'].mul(x['Fleet_value'], axis='index') + x['value'] = x['PcMoped'].mul(x['Fleet_value'] * x['aadt'], axis='index') else: x['value'] = np.nan return x[['value']] df['Fleet_value'] = df.groupby('Fleet_Class').apply(update_by_class) - for link_id, aux_df in df.groupby('Link_ID'): aadt = round(aux_df['aadt'].min(), 1) fleet_value = round(aux_df['Fleet_value'].sum(), 1) -- GitLab From 63188d33cfae6c0d62d0a9a59cc49fedf7549a01 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 20 Aug 2019 14:34:55 +0200 Subject: [PATCH 06/11] -Grid rotated: corrected bug when creating shape --- hermesv3_bu/grids/grid_rotated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/grids/grid_rotated.py b/hermesv3_bu/grids/grid_rotated.py index 02237f9..2195707 100755 --- a/hermesv3_bu/grids/grid_rotated.py +++ b/hermesv3_bu/grids/grid_rotated.py @@ -43,7 +43,7 @@ class RotatedGrid(Grid): # Initialises with parent class super(RotatedGrid, self).__init__(logger, attributes, auxiliary_path, vertical_description_path) - self.shape = (tstep_num, len(self.vertical_desctiption), len(attributes['rlat']), len(attributes['rlon'])) + self.shape = (tstep_num, len(self.vertical_desctiption), attributes['n_lat'], attributes['n_lon']) self.logger.write_time_log('RotatedGrid', '__init__', timeit.default_timer() - spent_time, 3) def create_regular_rotated(self): -- GitLab From b59169fb9c4cd9a04d11c4e2add725a753b77dc3 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 20 Aug 2019 14:35:31 +0200 Subject: [PATCH 07/11] -Aviation: corrected bug --- hermesv3_bu/sectors/aviation_sector.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hermesv3_bu/sectors/aviation_sector.py b/hermesv3_bu/sectors/aviation_sector.py index a280219..3bc15a0 100755 --- a/hermesv3_bu/sectors/aviation_sector.py +++ b/hermesv3_bu/sectors/aviation_sector.py @@ -485,11 +485,10 @@ class AviationSector(Sector): runway_shapefile.to_crs(self.grid_shp.crs, inplace=True) runway_shapefile['length'] = runway_shapefile.length # duplicating each runway by involved cell - runway_shapefile = gpd.sjoin(runway_shapefile.reset_index(), self.grid_shp, how="inner", + runway_shapefile = gpd.sjoin(runway_shapefile.reset_index(), self.grid_shp.reset_index(), how="inner", op='intersects') # Adding cell geometry - runway_shapefile = runway_shapefile.merge(self.grid_shp.reset_index().loc[:, ['FID', 'geometry']], - on='FID', how='left') + runway_shapefile = runway_shapefile.merge(self.grid_shp.reset_index(), on='FID', how='left') # Intersection between line (roadway) and polygon (cell) # runway_shapefile['geometry'] = runway_shapefile.apply(do_intersection, axis=1) runway_shapefile['mini_length'] = runway_shapefile.apply(get_intersection_length, axis=1) -- GitLab From ad977980976fab67406772f4397fa91012fd63d6 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 20 Aug 2019 14:53:20 +0200 Subject: [PATCH 08/11] - Writer default: Added compression for serial writing --- hermesv3_bu/writer/default_writer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 65c787a..b25d1ae 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -198,14 +198,16 @@ class DefaultWriter(Writer): # emissions.drop(columns=['Unnamed: 0'], inplace=True) for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - if CHUNK: - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, - chunksizes=self.rank_distribution[0]['shape']) - else: - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if CHUNK: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + var.set_collective(True) + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, -- GitLab From 88543b3470902a7c6492756ee4d8a402342d9f8f Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 20 Aug 2019 14:55:24 +0200 Subject: [PATCH 09/11] - Writer: Added compression for serial writing --- hermesv3_bu/writer/cmaq_writer.py | 4 +++- hermesv3_bu/writer/monarch_writer.py | 6 ++++-- hermesv3_bu/writer/wrfchem_writer.py | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 4670e6f..2d12b16 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -295,9 +295,11 @@ class CmaqWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) if self.comm_write.Get_size() > 1: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) var.set_collective(True) + else: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index 8981d51..44e08d1 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -192,11 +192,13 @@ class MonarchWriter(Writer): # var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, # chunksizes=self.rank_distribution[0]['shape']) - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) if self.comm_write.Get_size() > 1: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) var.set_collective(True) - + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True) + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, diff --git a/hermesv3_bu/writer/wrfchem_writer.py b/hermesv3_bu/writer/wrfchem_writer.py index 2d24d01..54d5c05 100755 --- a/hermesv3_bu/writer/wrfchem_writer.py +++ b/hermesv3_bu/writer/wrfchem_writer.py @@ -351,10 +351,13 @@ class WrfChemWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - var = netcdf.createVariable(var_name, np.float64, ('Time', 'emissions_zdim', 'south_north', 'west_east',)) - if self.comm_write.Get_size() > 1: + var = netcdf.createVariable(var_name, np.float64, + ('Time', 'emissions_zdim', 'south_north', 'west_east',)) var.set_collective(True) + else: + var = netcdf.createVariable(var_name, np.float64, + ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) -- GitLab From ed9fea5e1aef4f195f7ead986d0ec184a3b1807e Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 20 Aug 2019 16:45:44 +0200 Subject: [PATCH 10/11] PEP8 corrections --- hermesv3_bu/writer/monarch_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index 44e08d1..a7c6122 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -198,7 +198,7 @@ class MonarchWriter(Writer): var.set_collective(True) else: var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True) - + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, -- GitLab From f071efb9b2e2d6834e388d2533b086039062d76e Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 20 Aug 2019 16:46:11 +0200 Subject: [PATCH 11/11] PEP8 corrections --- hermesv3_bu/writer/monarch_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index a7c6122..20ee47b 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -198,7 +198,7 @@ class MonarchWriter(Writer): var.set_collective(True) else: var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True) - + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, -- GitLab