diff --git a/conf/hermes.conf b/conf/hermes.conf index 6088cfe12d5f582fda58e52207976e0d5c06393e..f3df9174f07d323e7298c3a607169687a2102964 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -3,20 +3,20 @@ log_level = 3 input_dir = /home/Earth/ctena/Models/hermesv3_bu_data data_path = /esarchive/recon output_dir = /scratch/Earth/HERMESv3_BU_OUT -output_name = HERMES_.nc +output_name = HERMESv3__rot.nc emission_summary = 1 start_date = 2016/11/29 00:00:00 # ----- end_date = start_date [DEFAULT] ----- # end_date = 2010/01/01 00:00:00 output_timestep_num = 25 -auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/py3/_ +auxiliary_files_path = /scratch/Earth/HERMESv3_BU_aux/_ erase_auxiliary_files = 0 [DOMAIN] # domain_type=[lcc, rotated, mercator, regular] -domain_type = lcc +domain_type = rotated # output_type=[MONARCH, CMAQ, WRF_CHEM, DEFAULT] output_model = DEFAULT output_attributes = /writing/global_attributes_WRF-Chem.csv @@ -24,12 +24,19 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver #vertical_description = /profiles/vertical/CMAQ_15layers_vertical_description.csv # if domain_type == rotated: - centre_lat = 51 - centre_lon = 10 - west_boundary = -35 - south_boundary = -27 - inc_rlat = 0.2 - inc_rlon = 0.2 + # centre_lat = 51 + # centre_lon = 10 + # west_boundary = -35 + # south_boundary = -27 + # inc_rlat = 0.2 + # inc_rlon = 0.2 + + centre_lat = 40.5 + centre_lon = -3.5 + west_boundary = -7.0 + south_boundary = -7.0 + inc_rlat = 0.4 + inc_rlon = 0.4 # if domain_type == lcc: @@ -117,16 +124,28 @@ vertical_description = /profiles/vertical/MONARCH_Global_48layers_ver [SECTOR MANAGEMENT] writing_processors = 12 +# aviation_processors = 1 +# shipping_port_processors = 1 +# livestock_processors = 12 +# crop_operations_processors = 1 +# crop_fertilizers_processors = 4 +# agricultural_machinery_processors = 1 +# residential_processors = 4 +# recreational_boats_processors = 4 +# point_sources_processors = 16 +# traffic_processors = 256 +# traffic_area_processors = 1 + aviation_processors = 0 shipping_port_processors = 0 livestock_processors = 0 crop_operations_processors = 0 crop_fertilizers_processors = 0 agricultural_machinery_processors = 0 -residential_processors = 1 +residential_processors = 0 recreational_boats_processors = 0 point_sources_processors = 0 -traffic_processors = 0 +traffic_processors = 1 traffic_area_processors = 0 @@ -299,7 +318,7 @@ resuspension_correction = 1 write_rline = 0 traffic_pollutants = nox_no2, nh3, co, so2, pm, voc, ch4 -vehicle_types = PCD_13 PCD_14 PCD_15 PCG_25 PCG_26 PCG_27 +# vehicle_types = PCD_13 PCD_14 PCD_15 PCG_25 PCG_26 PCG_27 # load = [0, 0.5, 1] load = 0.5 road_link_path = /traffic/road_links/2015/road_links_2015.shp @@ -322,7 +341,7 @@ traffic_speciation_profile_resuspension = /profiles/speciation/traffi traffic_area_pollutants = nox_no2,nmvoc,so2,co,nh3,pm10,pm25 do_evaporative = 1 traffic_area_gas_path = /traffic_area/gasoline_vehicles_provinces_2015.csv -popullation_by_municipality = /traffic_area/population_by_mun.csv +population_by_municipality = /traffic_area/population_by_mun.csv traffic_area_speciation_profiles_evaporative = /profiles/speciation/traffic_area/evaporative_cmaq.csv traffic_area_evaporative_ef_file = /traffic_area/ef/evaporative_nmvoc.csv do_small_cities = 1 diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index 3eab46da56fd30a8912695f25840f311567b9bbc..5ed5bd7ce31dccdb04bf83bd58eac4544159cf34 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -353,7 +353,7 @@ class Config(ArgParser): p.add_argument('--traffic_area_pollutants', required=False, help='...') p.add_argument('--do_evaporative', required=False, help='...') p.add_argument('--traffic_area_gas_path', required=False, help='...') - p.add_argument('--popullation_by_municipality', required=False, help='...') + p.add_argument('--population_by_municipality', required=False, help='...') p.add_argument('--traffic_area_speciation_profiles_evaporative', required=False, help='...') p.add_argument('--traffic_area_evaporative_ef_file', required=False, help='...') p.add_argument('--do_small_cities', required=False, help='...') diff --git a/hermesv3_bu/grids/grid_latlon.py b/hermesv3_bu/grids/grid_latlon.py index 8e1cd95a59c9132c9adca24efda28e1d70951150..d1c0513d8cf5b56a28c076ae877fbfb9f876c237 100755 --- a/hermesv3_bu/grids/grid_latlon.py +++ b/hermesv3_bu/grids/grid_latlon.py @@ -1,23 +1,5 @@ #!/usr/bin/env python -# Copyright 2018 Earth Sciences Department, BSC-CNS -# -# This file is part of HERMESv3_GR. -# -# HERMESv3_GR is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# HERMESv3_GR is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with HERMESv3_GR. If not, see . - - import os import timeit diff --git a/hermesv3_bu/grids/grid_mercator.py b/hermesv3_bu/grids/grid_mercator.py index f03ccd74cf025b8521614350bc49a9ab05144593..24faf7985e0b7057c39c4c1ea37d7300e2736cc9 100755 --- a/hermesv3_bu/grids/grid_mercator.py +++ b/hermesv3_bu/grids/grid_mercator.py @@ -4,7 +4,7 @@ import os import timeit import numpy as np from pyproj import Proj -from grid import Grid +from hermesv3_bu.grids.grid import Grid from hermesv3_bu.logger.log import Log diff --git a/hermesv3_bu/grids/grid_rotated.py b/hermesv3_bu/grids/grid_rotated.py index f8b1a1de6a771cf01f5e89ffae8e936cbb5d0d6d..219570769a3dbd3dc6c1dedaa5bbe5c939bc8dc8 100755 --- a/hermesv3_bu/grids/grid_rotated.py +++ b/hermesv3_bu/grids/grid_rotated.py @@ -2,7 +2,7 @@ import os import timeit -from grid import Grid +from hermesv3_bu.grids.grid import Grid import numpy as np import math @@ -43,7 +43,7 @@ class RotatedGrid(Grid): # Initialises with parent class super(RotatedGrid, self).__init__(logger, attributes, auxiliary_path, vertical_description_path) - self.shape = (tstep_num, len(self.vertical_desctiption), len(attributes['rlat']), len(attributes['rlon'])) + self.shape = (tstep_num, len(self.vertical_desctiption), attributes['n_lat'], attributes['n_lon']) self.logger.write_time_log('RotatedGrid', '__init__', timeit.default_timer() - spent_time, 3) def create_regular_rotated(self): diff --git a/hermesv3_bu/sectors/aviation_sector.py b/hermesv3_bu/sectors/aviation_sector.py index a2802196172121458d5416fe1fdea9e2eb19634a..3bc15a0fc7e4a07f6b972089ab2e6efc7090bffb 100755 --- a/hermesv3_bu/sectors/aviation_sector.py +++ b/hermesv3_bu/sectors/aviation_sector.py @@ -485,11 +485,10 @@ class AviationSector(Sector): runway_shapefile.to_crs(self.grid_shp.crs, inplace=True) runway_shapefile['length'] = runway_shapefile.length # duplicating each runway by involved cell - runway_shapefile = gpd.sjoin(runway_shapefile.reset_index(), self.grid_shp, how="inner", + runway_shapefile = gpd.sjoin(runway_shapefile.reset_index(), self.grid_shp.reset_index(), how="inner", op='intersects') # Adding cell geometry - runway_shapefile = runway_shapefile.merge(self.grid_shp.reset_index().loc[:, ['FID', 'geometry']], - on='FID', how='left') + runway_shapefile = runway_shapefile.merge(self.grid_shp.reset_index(), on='FID', how='left') # Intersection between line (roadway) and polygon (cell) # runway_shapefile['geometry'] = runway_shapefile.apply(do_intersection, axis=1) runway_shapefile['mini_length'] = runway_shapefile.apply(get_intersection_length, axis=1) diff --git a/hermesv3_bu/sectors/sector_manager.py b/hermesv3_bu/sectors/sector_manager.py index 51b355d9b305667d43ad56091414a8075daf8b92..b306dca36ff8caea46bcd11aa26100554a367aef 100755 --- a/hermesv3_bu/sectors/sector_manager.py +++ b/hermesv3_bu/sectors/sector_manager.py @@ -196,7 +196,7 @@ class SectorManager(object): arguments.auxiliary_files_path, grid.shapefile, clip, date_array, arguments.traffic_area_pollutants, grid.vertical_desctiption, arguments.population_density_map, arguments.speciation_map, arguments.molecular_weights, arguments.do_evaporative, arguments.traffic_area_gas_path, - arguments.popullation_by_municipality, arguments.nut_shapefile_prov, + arguments.population_by_municipality, arguments.nut_shapefile_prov, arguments.traffic_area_speciation_profiles_evaporative, arguments.traffic_area_evaporative_ef_file, arguments.temperature_hourly_files_path, arguments.do_small_cities, arguments.traffic_area_small_cities_path, arguments.traffic_area_speciation_profiles_small_cities, diff --git a/hermesv3_bu/sectors/traffic_sector.py b/hermesv3_bu/sectors/traffic_sector.py index 72f2acdd2af5b4094ac5ae54fb7a794f240bb567..27636b2c3ff081ca56f78762209b4149344113d6 100755 --- a/hermesv3_bu/sectors/traffic_sector.py +++ b/hermesv3_bu/sectors/traffic_sector.py @@ -255,8 +255,10 @@ class TrafficSector(Sector): df.drop(columns=['Adminis', 'CCAA', 'NETWORK_ID', 'Province', 'Road_name', 'aadt_m_sat', 'aadt_m_sun', 'aadt_m_wd', 'Source'], inplace=True) libc.malloc_trim(0) + # df.to_file('~/temp/road_links.shp') df = gpd.sjoin(df, self.clip.shapefile.to_crs(df.crs), how="inner", op='intersects') - df.drop(columns=['index_right', 'FID'], inplace=True) + # df.to_file('~/temp/road_links_selected.shp') + df.drop(columns=['index_right'], inplace=True) libc.malloc_trim(0) # Filtering road links to CONSiderate. @@ -436,17 +438,16 @@ class TrafficSector(Sector): if x.name == 'light_veh': x['value'] = x['PcLight'].mul(x['Fleet_value'] * x['aadt'], axis='index') elif x.name == 'heavy_veh': - x['value'] = x['PcHeavy'].mul(x['Fleet_value'], axis='index') + x['value'] = x['PcHeavy'].mul(x['Fleet_value'] * x['aadt'], axis='index') elif x.name == 'motos': - x['value'] = x['PcMoto'].mul(x['Fleet_value'], axis='index') + x['value'] = x['PcMoto'].mul(x['Fleet_value'] * x['aadt'], axis='index') elif x.name == 'mopeds': - x['value'] = x['PcMoped'].mul(x['Fleet_value'], axis='index') + x['value'] = x['PcMoped'].mul(x['Fleet_value'] * x['aadt'], axis='index') else: x['value'] = np.nan return x[['value']] df['Fleet_value'] = df.groupby('Fleet_Class').apply(update_by_class) - for link_id, aux_df in df.groupby('Link_ID'): aadt = round(aux_df['aadt'].min(), 1) fleet_value = round(aux_df['Fleet_value'].sum(), 1) diff --git a/hermesv3_bu/writer/cmaq_writer.py b/hermesv3_bu/writer/cmaq_writer.py index 0649c89181da5f3e440ffd21b2eefa6681a3c096..2d12b16261ca57758e23a1b0906c8ee71a5fa602 100755 --- a/hermesv3_bu/writer/cmaq_writer.py +++ b/hermesv3_bu/writer/cmaq_writer.py @@ -268,7 +268,11 @@ class CmaqWriter(Writer): """ spent_time = timeit.default_timer() - netcdf = Dataset(self.netcdf_path, mode='w', parallel=True, comm=self.comm_write, info=MPI.Info()) + if self.comm_write.Get_size() > 1: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w', parallel=True, comm=self.comm_write, + info=MPI.Info()) + else: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w') # ===== DIMENSIONS ===== self.logger.write_log('\tCreating NetCDF dimensions', message_level=2) @@ -291,8 +295,14 @@ class CmaqWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) + if self.comm_write.Get_size() > 1: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) + var.set_collective(True) + else: + var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=True) + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) - var = netcdf.createVariable(var_name, np.float64, ('TSTEP', 'LAY', 'ROW', 'COL',)) + var[:, :, self.rank_distribution[self.comm_write.Get_rank()]['y_min']: self.rank_distribution[self.comm_write.Get_rank()]['y_max'], diff --git a/hermesv3_bu/writer/default_writer.py b/hermesv3_bu/writer/default_writer.py index 65c787ab990d112da1a46c5b84ed7fc2ef92c0d9..b25d1aead2eaec2dacf5c10ab51dd9a0e6d8f60b 100755 --- a/hermesv3_bu/writer/default_writer.py +++ b/hermesv3_bu/writer/default_writer.py @@ -198,14 +198,16 @@ class DefaultWriter(Writer): # emissions.drop(columns=['Unnamed: 0'], inplace=True) for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - if CHUNK: - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, - chunksizes=self.rank_distribution[0]['shape']) - else: - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) - if self.comm_write.Get_size() > 1: + if CHUNK: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, + chunksizes=self.rank_distribution[0]['shape']) + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + var.set_collective(True) + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True) var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, diff --git a/hermesv3_bu/writer/monarch_writer.py b/hermesv3_bu/writer/monarch_writer.py index 7a00324e5c14c92f35110ca794fa642f6674f35e..20ee47badee3d57a2b1fb91f6c5470075dcfe97e 100755 --- a/hermesv3_bu/writer/monarch_writer.py +++ b/hermesv3_bu/writer/monarch_writer.py @@ -113,7 +113,11 @@ class MonarchWriter(Writer): """ from cf_units import Unit spent_time = timeit.default_timer() - netcdf = Dataset(self.netcdf_path, mode='w', parallel=True, comm=self.comm_write, info=MPI.Info()) + if self.comm_write.Get_size() > 1: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w', parallel=True, comm=self.comm_write, + info=MPI.Info()) + else: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w') # ========== DIMENSIONS ========== self.logger.write_log('\tCreating NetCDF dimensions', message_level=2) @@ -186,10 +190,16 @@ class MonarchWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) - var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) # var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, # chunksizes=self.rank_distribution[0]['shape']) - var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + + if self.comm_write.Get_size() > 1: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim) + var.set_collective(True) + else: + var = netcdf.createVariable(var_name, np.float64, ('time', 'lev',) + var_dim, zlib=True) + + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) var[:, :, self.rank_distribution[self.comm_write.Get_rank()]['y_min']: diff --git a/hermesv3_bu/writer/wrfchem_writer.py b/hermesv3_bu/writer/wrfchem_writer.py index 4943f99c58a75604255a139e01d2d33b3d671736..54d5c0528374ad46bfe26dc370b3b6bde7a1b784 100755 --- a/hermesv3_bu/writer/wrfchem_writer.py +++ b/hermesv3_bu/writer/wrfchem_writer.py @@ -327,7 +327,11 @@ class WrfChemWriter(Writer): """ spent_time = timeit.default_timer() - netcdf = Dataset(self.netcdf_path, mode='w', parallel=True, comm=self.comm_write, info=MPI.Info()) + if self.comm_write.Get_size() > 1: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w', parallel=True, comm=self.comm_write, + info=MPI.Info()) + else: + netcdf = Dataset(self.netcdf_path, format="NETCDF4", mode='w') # ===== DIMENSIONS ===== self.logger.write_log('\tCreating NetCDF dimensions', message_level=2) @@ -347,8 +351,16 @@ class WrfChemWriter(Writer): for var_name in emissions.columns.values: self.logger.write_log('\t\tCreating {0} variable'.format(var_name), message_level=3) + if self.comm_write.Get_size() > 1: + var = netcdf.createVariable(var_name, np.float64, + ('Time', 'emissions_zdim', 'south_north', 'west_east',)) + var.set_collective(True) + else: + var = netcdf.createVariable(var_name, np.float64, + ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=True) + var_data = self.dataframe_to_array(emissions.loc[:, [var_name]]) - var = netcdf.createVariable(var_name, np.float64, ('Time', 'emissions_zdim', 'south_north', 'west_east',)) + var[:, :, self.rank_distribution[self.comm_write.Get_rank()]['y_min']: self.rank_distribution[self.comm_write.Get_rank()]['y_max'],