From 8208a6ceb5544bd77236c601ceeff13e0b6fa25d Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 13 Mar 2019 18:21:22 +0100 Subject: [PATCH 01/10] Added barrier when writing the grid shapefile to make sure that all the process have their own grid shapefile. --- hermesv3_gr/modules/grids/grid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 9263e64..f090e7e 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -531,6 +531,7 @@ class Grid(object): settings.write_log('\t\tGrid shapefile already done. Lets try to read it.', level=3) gdf = gpd.read_file(self.shapefile_path) + settings.comm.Barrier() settings.write_time('Grid', 'to_shapefile', timeit.default_timer() - st_time, level=1) return gdf -- GitLab From b226bbd9ef7a38feb51412fe3ca969dda25c3e34 Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 13 Mar 2019 18:24:44 +0100 Subject: [PATCH 02/10] Added barrier when creating shapefile folder --- hermesv3_gr/modules/grids/grid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index f090e7e..2669dd1 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -411,6 +411,7 @@ class Grid(object): if settings.rank == 0: if not os.path.exists(self.shapefile_path): os.makedirs(self.shapefile_path) + settings.comm.Barrier() if full_grid: self.shapefile_path = os.path.join(self.shapefile_path, 'grid_shapefile.shp') else: -- GitLab From e2f74877d77fd422c46c662e9e2c0d12ba2c093a Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 19 Mar 2019 17:14:07 +0100 Subject: [PATCH 03/10] changed input_dir by data_path in the parse method for the gridded profiles --- .../modules/emision_inventories/emission_inventory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index 2ce190d..ca38e50 100644 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -317,25 +317,25 @@ class EmissionInventory(object): #MONTHLY try: # gridded temporal profile - p_month = emission_inventory.p_month.replace('', options.input_dir) + p_month = emission_inventory.p_month.replace('', options.data_path) except AttributeError: p_month = emission_inventory.p_month # WEEKLY try: # gridded temporal profile - p_week = emission_inventory.p_week.replace('', options.input_dir) + p_week = emission_inventory.p_week.replace('', options.data_path) except AttributeError: p_week = emission_inventory.p_week # DAILY try: # gridded temporal profile - p_day = emission_inventory.p_day.replace('', options.input_dir) + p_day = emission_inventory.p_day.replace('', options.data_path) except AttributeError: p_day = emission_inventory.p_day # HOURLY try: # gridded temporal profile - p_hour = emission_inventory.p_hour.replace('', options.input_dir) + p_hour = emission_inventory.p_hour.replace('', options.data_path) except AttributeError: p_hour = emission_inventory.p_hour -- GitLab From 1a6c0a656bf1904a675b6101651bd2689ca2c0f2 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 26 Mar 2019 16:49:53 +0100 Subject: [PATCH 04/10] Corrected bug on parallel NetCDF metadata on rotated domain --- hermesv3_gr/modules/writing/writer_monarch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index 3321b06..e501926 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -369,8 +369,8 @@ class WriterMonarch(Writer): # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' - mapping.grid_north_pole_latitude = self.grid.new_pole_latitude_degrees - mapping.grid_north_pole_longitude = 90 - self.grid.new_pole_longitude_degrees + mapping.grid_north_pole_latitude = 90 - self.grid.new_pole_latitude_degrees + mapping.grid_north_pole_longitude = self.grid.new_pole_longitude_degrees elif LambertConformalConic: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') -- GitLab From 4a7bc1b79079f4a45d2bd052e98c09bd312359ec Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 28 Mar 2019 17:36:31 +0100 Subject: [PATCH 05/10] Testing --- CHANGELOG | 5 + conf/hermes.conf | 18 ++- hermesv3_gr/config/config.py | 12 +- hermesv3_gr/hermes.py | 3 +- hermesv3_gr/modules/grids/grid.py | 10 +- hermesv3_gr/modules/grids/grid_latlon.py | 139 +++++++++++++++++++++++ 6 files changed, 176 insertions(+), 11 deletions(-) create mode 100644 hermesv3_gr/modules/grids/grid_latlon.py diff --git a/CHANGELOG b/CHANGELOG index e7abef4..f37ad92 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +1.X.X + + - Corrected error on MONARCH rotated NetCDF metadata + - Regional regular lat-lon + 1.0.1 2019/03/12 diff --git a/conf/hermes.conf b/conf/hermes.conf index 56d26d2..68ce6b4 100644 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -20,8 +20,9 @@ output_model = MONARCH # output_model = WRF_CHEM output_attributes = /data/global_attributes.csv -# ***** domain_type=[global, lcc, rotated, mercator] ***** -domain_type = global +# ***** domain_type=[global, regular-lat-lon, lcc, rotated, mercator] ***** +#domain_type = global +domain_type = regular-lat-lon # domain_type = lcc # domain_type = rotated # domain_type = mercator @@ -29,8 +30,17 @@ vertical_description = /data/profiles/vertical/Benchmark_15layers_ver auxiliar_files_path = /data/auxiliar_files/_ # if domain_type == global: - inc_lat = 1. - inc_lon = 1.40625 + #inc_lat = 1. + #inc_lon = 1.40625 + +# if domain_type == regular-lat-lon: + lat_orig = 30. + lon_orig = -30. + inc_lat = 0.05 + inc_lon = 0.1 + n_lat = 840 + n_lon = 900 + # if domain_type == rotated: #centre_lat = 35 diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index b61857d..9ce12d6 100644 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -19,7 +19,7 @@ from configargparse import ArgParser - +import sys class Config(ArgParser): """ @@ -67,7 +67,7 @@ class Config(ArgParser): help='Path to the file that contains the global attributes.') p.add_argument('--domain_type', required=True, help='Type of domain to simulate.', - choices=['global', 'lcc', 'rotated', 'mercator']) + choices=['global', 'lcc', 'rotated', 'mercator', 'regular-lat-lon']) p.add_argument('--auxiliar_files_path', required=True, help='Path to the directory where the necessary auxiliary files will be created if them are ' + 'not created yet.') @@ -79,6 +79,12 @@ class Config(ArgParser): p.add_argument('--inc_lat', required=False, help='Latitude resolution for a global domain.', type=float) p.add_argument('--inc_lon', required=False, help='Longitude resolution for a global domain.', type=float) + # Regular lat-lon options: + p.add_argument('--lat_orig', required=False, help='Latitude of the corner of the first cell.', type=float) + p.add_argument('--lon_orig', required=False, help='Longitude of the corner of the first cell.', type=float) + p.add_argument('--n_lat', required=False, help='Number of latitude elements.', type=float) + p.add_argument('--n_lon', required=False, help='Number of longitude elements.', type=float) + # Rotated options p.add_argument('--centre_lat', required=False, help='Central geographic latitude of grid (non-rotated degrees). Corresponds to the TPH0D ' + @@ -157,7 +163,7 @@ class Config(ArgParser): if is_str: exec("options.{0} = options.{0}.replace('', options.input_dir)".format(item)) exec("options.{0} = options.{0}.replace('', options.domain_type)".format(item)) - if options.domain_type == 'global': + if options.domain_type == 'global' or options.domain_type == 'regular-lat-lon': exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( item, options.inc_lat, options.inc_lon)) elif options.domain_type == 'rotated': diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index f094111..52d46cb 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -65,7 +65,8 @@ class Hermes(object): self.grid = Grid.select_grid( self.options.domain_type, self.options.vertical_description, self.options.output_timestep_num, - self.options.auxiliar_files_path, self.options.inc_lat, self.options.inc_lon, self.options.centre_lat, + self.options.auxiliar_files_path, self.options.inc_lat, self.options.inc_lon, self.options.lat_orig, + self.options.lon_orig, self.options.n_lat, self.options.n_lon, self.options.centre_lat, self.options.centre_lon, self.options.west_boundary, self.options.south_boundary, self.options.inc_rlat, self.options.inc_rlon, self.options.lat_1, self.options.lat_2, self.options.lon_0, self.options.lat_0, self.options.nx, self.options.ny, self.options.inc_x, self.options.inc_y, self.options.x_0, diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 2669dd1..b6e5a29 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -94,8 +94,8 @@ class Grid(object): return grid @staticmethod - def select_grid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, - centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon, + def select_grid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, + lon_orig, n_lat, n_lon, centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon, lat_1, lat_2, lon_0, lat_0, nx, ny, inc_x, inc_y, x_0, y_0, lat_ts): # TODO describe better the rotated parameters """ @@ -182,6 +182,10 @@ class Grid(object): from hermesv3_gr.modules.grids.grid_global import GlobalGrid grid = GlobalGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon) + elif grid_type == 'regular-lat-lon': + from hermesv3_gr.modules.grids.grid_latlon import LatLonGrid + grid = LatLonGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, + lat_orig, lon_orig, n_lat, n_lon) elif grid_type == 'rotated': from hermesv3_gr.modules.grids.grid_rotated import RotatedGrid grid = RotatedGrid(grid_type, vertical_description_path, timestep_num, temporal_path, @@ -200,7 +204,7 @@ class Grid(object): settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise NotImplementedError("The grid type {0} is not implemented.".format(grid_type) - + " Use 'global', 'rotated' or 'lcc'.") + + " Use 'global', 'regular-lat-lon, 'rotated', 'lcc' or 'mercator.") sys.exit(1) settings.write_time('Grid', 'select_grid', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/grids/grid_latlon.py b/hermesv3_gr/modules/grids/grid_latlon.py new file mode 100644 index 0000000..19d21cf --- /dev/null +++ b/hermesv3_gr/modules/grids/grid_latlon.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +# Copyright 2018 Earth Sciences Department, BSC-CNS +# +# This file is part of HERMESv3_GR. +# +# HERMESv3_GR is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HERMESv3_GR is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with HERMESv3_GR. If not, see . + + +import os +import sys +import timeit + +import hermesv3_gr.config.settings as settings +from grid import Grid + + +class LatLonGrid(Grid): + """ + Regional regular lat-lon grid object that contains all the information to do a global output. + + :param grid_type: Type of the output grid [global, rotated, lcc, mercator]. + :type grid_type: str + + :param vertical_description_path: Path to the file that contains the vertical description. + :type vertical_description_path: str + + :param timestep_num: Number of timesteps. + :type timestep_num: int + + :param temporal_path: Path to the temporal folder. + :type temporal_path: str + + :param inc_lat: Increment between latitude centroids. + :type inc_lat: float + + :param inc_lon: Increment between longitude centroids. + :type inc_lon: float + + :param lat_orig: Location of the latitude of the center of the first cell (down left). + :type lat_orig: float + + :param lon_orig: Location of the longitude of the center of the first cell (down left). + :type lon_orig: float + + :param n_lat: Number of cells on the latitude direction. + :type n_lat = int + + :param n_lon: Number of cells on the latitude direction. + :type n_lon = int + + """ + + def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, + lon_orig, n_lat, n_lon): + import ESMF + + st_time = timeit.default_timer() + settings.write_log('\tCreating Regional regular lat-lon grid.', level=2) + + # Initialize the class using parent + super(LatLonGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + + self.inc_lat = inc_lat + self.inc_lon = inc_lon + self.lat_orig = lat_orig + self.lon_orig = lon_orig + self.n_lat = n_lat + self.n_lon = n_lon + + self.crs = {'init': 'epsg:4326'} + self.create_coords() + + if not os.path.exists(self.coords_netcdf_file): + if settings.rank == 0: + self.write_coords_netcdf() + settings.comm.Barrier() + + self.esmf_grid = self.create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) + + self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] + self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] + self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] + self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + + self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, + self.y_upper_bound-self.y_lower_bound) + settings.comm.Barrier() + + total_area = self.get_cell_area() + self.full_shape = (timestep_num, len(self.vertical_description), total_area.shape[-2], total_area.shape[-1]) + self.cell_area = total_area[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound] + + settings.write_time('LatLonGrid', 'Init', timeit.default_timer() - st_time, level=1) + + def create_coords(self): + """ + Create the coordinates for a global domain. + """ + import numpy as np + + st_time = timeit.default_timer() + settings.write_log('\t\tCreating global coordinates', level=3) + + # From corner latitude /longitude to center ones + lat_c_orig = self.lat_orig + (self.inc_lat / 2) + lon_c_orig = self.lon_orig + (self.inc_lon / 2) + + self.center_latitudes = np.arange(lat_c_orig, lat_c_orig + self.inc_lat * self.n_lat, self.inc_lat, + dtype=np.float) + self.boundary_latitudes = self.create_bounds(self.center_latitudes, self.inc_lat) + + # ===== Longitudes ===== + self.center_longitudes = np.arange(lon_c_orig, lon_c_orig + self.inc_lon * self.n_lon, self.inc_lon, + dtype=np.float) + if len(self.center_longitudes)//2 < settings.size: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError("ERROR: Maximum number of processors exceeded. " + + "It has to be less or equal than {0}.".format(len(self.center_longitudes)//2)) + sys.exit(1) + self.boundary_longitudes = self.create_bounds(self.center_longitudes, self.inc_lon) + + settings.write_time('LatLonGrid', 'create_coords', timeit.default_timer() - st_time, level=2) + + +if __name__ == '__main__': + pass -- GitLab From 30c49855e7ce723f29f3d05aec555feea3a68e04 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 29 Mar 2019 11:24:04 +0100 Subject: [PATCH 06/10] Regional regular lat lon to test --- conf/hermes.conf | 6 +++--- hermesv3_gr/config/config.py | 4 ++-- hermesv3_gr/modules/grids/grid.py | 4 ++-- hermesv3_gr/modules/grids/grid_latlon.py | 7 +++++-- hermesv3_gr/modules/writing/writer_monarch.py | 4 ++-- hermesv3_gr/tools/netcdf_tools.py | 1 - 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 68ce6b4..8eb5610 100644 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -20,9 +20,9 @@ output_model = MONARCH # output_model = WRF_CHEM output_attributes = /data/global_attributes.csv -# ***** domain_type=[global, regular-lat-lon, lcc, rotated, mercator] ***** +# ***** domain_type=[global, regular, lcc, rotated, mercator] ***** #domain_type = global -domain_type = regular-lat-lon +domain_type = regular # domain_type = lcc # domain_type = rotated # domain_type = mercator @@ -33,7 +33,7 @@ auxiliar_files_path = /data/auxiliar_files/_ #inc_lat = 1. #inc_lon = 1.40625 -# if domain_type == regular-lat-lon: +# if domain_type == regular: lat_orig = 30. lon_orig = -30. inc_lat = 0.05 diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index 9ce12d6..ab62fd5 100644 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -67,7 +67,7 @@ class Config(ArgParser): help='Path to the file that contains the global attributes.') p.add_argument('--domain_type', required=True, help='Type of domain to simulate.', - choices=['global', 'lcc', 'rotated', 'mercator', 'regular-lat-lon']) + choices=['global', 'lcc', 'rotated', 'mercator', 'regular']) p.add_argument('--auxiliar_files_path', required=True, help='Path to the directory where the necessary auxiliary files will be created if them are ' + 'not created yet.') @@ -163,7 +163,7 @@ class Config(ArgParser): if is_str: exec("options.{0} = options.{0}.replace('', options.input_dir)".format(item)) exec("options.{0} = options.{0}.replace('', options.domain_type)".format(item)) - if options.domain_type == 'global' or options.domain_type == 'regular-lat-lon': + if options.domain_type == 'global' or options.domain_type == 'regular': exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( item, options.inc_lat, options.inc_lon)) elif options.domain_type == 'rotated': diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index b6e5a29..86b503b 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -182,7 +182,7 @@ class Grid(object): from hermesv3_gr.modules.grids.grid_global import GlobalGrid grid = GlobalGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon) - elif grid_type == 'regular-lat-lon': + elif grid_type == 'regular': from hermesv3_gr.modules.grids.grid_latlon import LatLonGrid grid = LatLonGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, lon_orig, n_lat, n_lon) @@ -204,7 +204,7 @@ class Grid(object): settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise NotImplementedError("The grid type {0} is not implemented.".format(grid_type) - + " Use 'global', 'regular-lat-lon, 'rotated', 'lcc' or 'mercator.") + + " Use 'global', 'regular, 'rotated', 'lcc' or 'mercator.") sys.exit(1) settings.write_time('Grid', 'select_grid', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/grids/grid_latlon.py b/hermesv3_gr/modules/grids/grid_latlon.py index 19d21cf..c8e2dea 100644 --- a/hermesv3_gr/modules/grids/grid_latlon.py +++ b/hermesv3_gr/modules/grids/grid_latlon.py @@ -119,7 +119,7 @@ class LatLonGrid(Grid): self.center_latitudes = np.arange(lat_c_orig, lat_c_orig + self.inc_lat * self.n_lat, self.inc_lat, dtype=np.float) - self.boundary_latitudes = self.create_bounds(self.center_latitudes, self.inc_lat) + boundary_latitudes = self.create_bounds(self.center_latitudes, self.inc_lat) # ===== Longitudes ===== self.center_longitudes = np.arange(lon_c_orig, lon_c_orig + self.inc_lon * self.n_lon, self.inc_lon, @@ -130,7 +130,10 @@ class LatLonGrid(Grid): raise AttributeError("ERROR: Maximum number of processors exceeded. " + "It has to be less or equal than {0}.".format(len(self.center_longitudes)//2)) sys.exit(1) - self.boundary_longitudes = self.create_bounds(self.center_longitudes, self.inc_lon) + boundary_longitudes = self.create_bounds(self.center_longitudes, self.inc_lon) + + self.boundary_latitudes = boundary_latitudes.reshape((1,) + boundary_latitudes.shape) + self.boundary_longitudes = boundary_longitudes.reshape((1,) + boundary_longitudes.shape) settings.write_time('LatLonGrid', 'create_coords', timeit.default_timer() - st_time, level=2) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index e501926..8749ed5 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -127,7 +127,7 @@ class WriterMonarch(Writer): RegularLatLon = False Rotated = False LambertConformalConic = False - if self.grid.grid_type == 'global': + if self.grid.grid_type == 'global' or self.grid.grid_type == 'regular': RegularLatLon = True elif self.grid.grid_type == 'rotated': Rotated = True @@ -428,7 +428,7 @@ class WriterMonarch(Writer): rotated = False lcc = False - if self.grid.grid_type == 'global': + if self.grid.grid_type == 'global' or self.grid.grid_type == 'regular': regular_latlon = True elif self.grid.grid_type == 'rotated': rotated = True diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index f0d72de..955468e 100644 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -271,7 +271,6 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, if boundary_longitudes is not None: lons.bounds = "lon_bnds" lon_bnds = netcdf.createVariable('lon_bnds', 'f', lon_dim + ('nv',), zlib=True) - # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes if rotated: -- GitLab From a130ba0f64ca1eb25be7219cdb40938635d3606e Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 1 Apr 2019 12:41:02 +0200 Subject: [PATCH 07/10] PEP8 corrections --- hermesv3_gr/config/config.py | 1 + .../emision_inventories/emission_inventory.py | 2 +- .../point_gfas_emission_inventory.py | 3 +- hermesv3_gr/modules/temporal/temporal.py | 172 +----------------- hermesv3_gr/modules/vertical/vertical_gfas.py | 1 - 5 files changed, 8 insertions(+), 171 deletions(-) diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index ab62fd5..d174e04 100644 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -21,6 +21,7 @@ from configargparse import ArgParser import sys + class Config(ArgParser): """ Initialization of the arguments that the parser can handle. diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index ca38e50..ed0a837 100644 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -314,7 +314,7 @@ class EmissionInventory(object): num += 1 pollutants = list(map(str, re.split(', |,|; |;| ', emission_inventory.pollutants))) - #MONTHLY + # MONTHLY try: # gridded temporal profile p_month = emission_inventory.p_month.replace('', options.data_path) diff --git a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py index e9a4c8e..0eb27d3 100755 --- a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py @@ -282,7 +282,8 @@ class PointGfasEmissionInventory(EmissionInventory): gdf[pollutant['name']] = aux * gdf['src_area'] # print 'masa {0}: {1} '.format(pollutant['name'], gdf[pollutant['name']].sum()) - gdf[pollutant['name']] = (aux / gdf['dst_area']) * netcdf.variables['cell_area'][:].flatten()[gdf['src_index']] + gdf[pollutant['name']] = (aux / gdf['dst_area']) * \ + netcdf.variables['cell_area'][:].flatten()[gdf['src_index']] # print netcdf.variables['bc'][:].sum() netcdf.close() diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 462380d..9ebb759 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -198,11 +198,12 @@ class TemporalDistribution(object): def get_gridded_temporal_profile(self, profile_value, profile_path): from netCDF4 import Dataset nc_in = Dataset(profile_path, mode='r') - if nc_in.dimensions['latitude'].size * nc_in.dimensions['longitude'].size != self.grid.full_shape[-2] * self.grid.full_shape[-1]: + if nc_in.dimensions['latitude'].size * nc_in.dimensions['longitude'].size != self.grid.full_shape[-2] * \ + self.grid.full_shape[-1]: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: - raise IOError('ERROR: The temporal gridded profile {0} is not in the destiny resolution like {1}.'.format( - profile_path, self.grid.coords_netcdf_file)) + raise IOError('ERROR: The temporal gridded profile {0} '.format(profile_path) + + 'is not in the destiny resolution like {1}.'.format(self.grid.coords_netcdf_file)) sys.exit(1) try: @@ -470,70 +471,6 @@ class TemporalDistribution(object): return profile - # def calculate_ending_date(self): - # """ - # Calculate the date of the last timestep. - # - # :return: Date of the last timestep - # :rtype: datetime.datetime - # """ - # from datetime import timedelta - # - # st_time = timeit.default_timer() - # - # if self.timestep_type == 'hourly': - # end_date = self.starting_date + (self.timestep_num - 1) * timedelta(hours=self.timestep_freq) - # elif self.timestep_type == 'daily': - # end_date = self.starting_date + (self.timestep_num - 1) * timedelta(hours=self.timestep_freq * 24) - # elif self.timestep_type == 'monthly': - # delta_year = (self.timestep_num - 1) * self.timestep_freq // 12 - # delta_month = (self.timestep_num - 1) * self.timestep_freq % 12 - # end_date = self.starting_date.replace(year=self.starting_date.year + delta_year, - # month=self.starting_date.month + delta_month) - # elif self.timestep_type == 'yearly': - # delta_year = (self.timestep_num - 1) * self.timestep_freq - # end_date = self.starting_date.replace(year=self.starting_date.year + delta_year) - # else: - # end_date = self.starting_date - # - # settings.write_time('TemporalDistribution', 'calculate_ending_date', timeit.default_timer() - st_time, level=3) - # - # return end_date - # - # def calculate_timedelta(self, date): - # """ - # Calculate the difference of time to the next timestep. - # - # :param date: Date of the current timestep. - # :type date: datetime.datetime - # - # :return: Difference of time to the next timestep. - # :rtype: datetime.timedelta - # """ - # from datetime import timedelta - # from calendar import monthrange, isleap - # - # st_time = timeit.default_timer() - # - # if self.timestep_type == 'hourly': - # delta = timedelta(hours=self.timestep_freq) - # elif self.timestep_type == 'daily': - # delta = timedelta(hours=self.timestep_freq * 24) - # elif self.timestep_type == 'monthly': - # days = monthrange(date.year, date.month)[1] - # delta = timedelta(hours=days * 24) - # elif self.timestep_type == 'yearly': - # if isleap(date.year): - # delta = timedelta(hours=366 * 24) - # else: - # delta = timedelta(hours=365 * 24) - # else: - # delta = None - # - # settings.write_time('TemporalDistribution', 'calculate_ending_date', timeit.default_timer() - st_time, level=3) - # - # return delta - def get_tz_from_id(self, tz_id): """ Extract the timezone (string format) for the given id (int). @@ -902,107 +839,6 @@ class TemporalDistribution(object): level=3) return factors - # def get_temporal_hourly_profile(self, profile_id, date=None): - # """ - # Extract the hourly profile of the given ID in a dictionary format. - # - # The hour (0 to 23) is the key (int) and the value (float) is the factor. - # - # :param profile_id: ID of the hourly profile to use. - # :type profile_id: str - # - # :param date: Date of the timestep to simulate. Not necessary for a single ID. - # :type date: datetime.datetime - # - # :return: Hourly profile where the hour (0 to 23) is the key (int) and the value (float) is the factor. - # :rtype: dict - # """ - # import pandas as pd - # - # st_time = timeit.default_timer() - # if date is None: - # df = pd.read_csv(self.hourly_profile_path) - # try: - # profile = df.loc[df[df.TP_H == profile_id].index[0]].to_dict() - # except IndexError: - # settings.write_log('ERROR: Check the .err file to get more info.') - # if settings.rank == 0: - # raise AttributeError('ERROR: Hourly profile ID {0} is not in the {1} file.'.format( - # profile_id, self.hourly_profile_path)) - # sys.exit(1) - # profile.pop('TP_H', None) - # profile = {int(k): float(v) for k, v in profile.items()} - # else: - # profile = None - # settings.write_time('TemporalDistribution', 'get_temporal_hourly_profile', timeit.default_timer() - st_time, - # level=3) - # - # return profile - # - # def get_temporal_weekly_profile(self, date): - # """ - # Extract the daily profile of the given ID in a dictionary format. - # - # The weekday (0 to 6) is the key (int) and the value (float) is the factor. - # - # :param date: Date of the timestep to simulate. - # :type date: datetime.datetime - # - # :return: Daily profile where the weekday (0 to 6) is the key (int) and the value (float) is the factor. - # :rtype: dict - # """ - # import pandas as pd - # - # st_time = timeit.default_timer() - # - # if self.weekly_profile_id is not None: - # df = pd.read_csv(self.weekly_profile_path) - # try: - # profile = df.loc[df[df.TP_W == self.weekly_profile_id].index[0]].to_dict() - # except IndexError: - # settings.write_log('ERROR: Check the .err file to get more info.') - # if settings.rank == 0: - # raise AttributeError('ERROR: Daily profile ID {0} is not in the {1} file.'.format( - # self.weekly_profile_id, self.weekly_profile_path)) - # sys.exit(1) - # profile.pop('TP_W', None) - # profile_aux = {int(k): float(v) for k, v in profile.items()} - # rebalance_factor = self.calculate_rebalance_factor(profile_aux, date) - # profile = {int(k): float(v) + rebalance_factor for k, v in profile.items()} - # else: - # profile = None - # - # settings.write_time('TemporalDistribution', 'get_temporal_weekly_profile', timeit.default_timer() - st_time, - # level=3) - # - # return profile - # - # def calculate_rebalance_factor(self, profile, date): - # """ - # Calculate the necessary factor make consistent the full month data. - # - # This is needed for the months that if you sum the daily factor of each day of the month it doesn't sum as - # the number of days of the month. - # - # :param profile: Daily profile. - # :type profile: dict - # - # :param date: Date of the timestep to simulate. - # :type date: datetime.datetime - # - # :return: Rebalance factor to be sum to the daily factor. - # :rtype: float - # """ - # st_time = timeit.default_timer() - # - # weekdays = self.calculate_weekdays(date) - # rebalance_factor = self.calculate_weekday_factor(profile, weekdays) - # - # settings.write_time('TemporalDistribution', 'calculate_rebalance_factor', timeit.default_timer() - st_time, - # level=3) - # - # return rebalance_factor - @staticmethod def calculate_weekday_factor(profile, weekdays): # TODO Documentation diff --git a/hermesv3_gr/modules/vertical/vertical_gfas.py b/hermesv3_gr/modules/vertical/vertical_gfas.py index 57da232..3090926 100644 --- a/hermesv3_gr/modules/vertical/vertical_gfas.py +++ b/hermesv3_gr/modules/vertical/vertical_gfas.py @@ -54,7 +54,6 @@ class GfasVerticalDistribution(VerticalDistribution): return string - @staticmethod def calculate_widths(heights_list): """ -- GitLab From 6f11b2c1c40f526ddf699f59f7efda249068a77f Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 2 Apr 2019 10:46:31 +0200 Subject: [PATCH 08/10] CHanged description --- hermesv3_gr/modules/grids/grid_latlon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermesv3_gr/modules/grids/grid_latlon.py b/hermesv3_gr/modules/grids/grid_latlon.py index c8e2dea..21b28ab 100644 --- a/hermesv3_gr/modules/grids/grid_latlon.py +++ b/hermesv3_gr/modules/grids/grid_latlon.py @@ -48,10 +48,10 @@ class LatLonGrid(Grid): :param inc_lon: Increment between longitude centroids. :type inc_lon: float - :param lat_orig: Location of the latitude of the center of the first cell (down left). + :param lat_orig: Location of the latitude of the corner of the first cell (down left). :type lat_orig: float - :param lon_orig: Location of the longitude of the center of the first cell (down left). + :param lon_orig: Location of the longitude of the corner of the first cell (down left). :type lon_orig: float :param n_lat: Number of cells on the latitude direction. -- GitLab From 0c0ef6955ba9e1460e404665e81a0639597bff62 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 26 Apr 2019 17:32:14 +0200 Subject: [PATCH 09/10] Solved error to create shapefile on regular lat lon domains --- hermesv3_gr/modules/grids/grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 86b503b..81c4dfd 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -433,7 +433,7 @@ class Grid(object): x = self.boundary_longitudes # sys.exit() - if self.grid_type == 'global': + if self.grid_type in ['global', 'regular']: x = x.reshape((x.shape[1], x.shape[2])) y = y.reshape((y.shape[1], y.shape[2])) -- GitLab From 2508ceb9bb5683eb876b9fbe60dea2940c77f9b0 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 30 Apr 2019 13:06:01 +0200 Subject: [PATCH 10/10] Ready to release --- CHANGELOG | 5 +++-- hermesv3_gr/__init__.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f37ad92..ed2436d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,8 @@ -1.X.X +1.0.2 + 2019/04/30 - Corrected error on MONARCH rotated NetCDF metadata - - Regional regular lat-lon + - Regional regular lat-lon domain 1.0.1 2019/03/12 diff --git a/hermesv3_gr/__init__.py b/hermesv3_gr/__init__.py index 5c4105c..7863915 100644 --- a/hermesv3_gr/__init__.py +++ b/hermesv3_gr/__init__.py @@ -1 +1 @@ -__version__ = "1.0.1" +__version__ = "1.0.2" -- GitLab