diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5126b139d78ddd60d97f79f27bb3119cd3b36f52..a91fafdc41f2aab1b2bc36e693cbb172dedac2fd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,7 @@ CHANGELOG * Improved load_nes.py removing redundant code * Fix vertical interpolation for descendant level values (Pressure) ([#71](https://earth.bsc.es/gitlab/es/NES/-/issues/71)) * Removed lat-lon dimension on the NetCDF projections that not need them + * Fixed the bug when creating the spatial bounds after selecting a region ([#68](https://earth.bsc.es/gitlab/es/NES/-/issues/68)) 1.1.3 ============ diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 0c64e6ba2f0d3e7def86e154c74d191b213055d6..965e103531aef7980634db1d4fec265c12475e3f 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -59,8 +59,12 @@ class Nes(object): Vertical level dictionary with the complete 'data' key for all the values and the rest of the attributes. _lat : dict Latitudes dictionary with the complete 'data' key for all the values and the rest of the attributes. - _lon _ dict + _lon : dict Longitudes dictionary with the complete 'data' key for all the values and the rest of the attributes. + _lat_bnds : None or dict + Latitude bounds dictionary with the complete 'data' key for the latitudinal boundaries of each grid and the rest of the attributes. + _lon_bnds : None or dict + Longitude bounds dictionary with the complete 'data' key for the longitudinal boundaries of each grid and the rest of the attributes. parallel_method : str Parallel method to read/write. Can be chosen any of the following axis to parallelize: 'T', 'Y' or 'X'. @@ -78,6 +82,10 @@ class Nes(object): Latitudes dictionary with the portion of 'data' corresponding to the rank values. lon : dict Longitudes dictionary with the portion of 'data' corresponding to the rank values. + lat_bnds : None or dict + Latitude bounds dictionary with the portion of 'data' for the latitudinal boundaries corresponding to the rank values. + lon_bnds : None or dict + Longitude bounds dictionary with the portion of 'data' for the longitudinal boundaries corresponding to the rank values. global_attrs : dict Global attributes with the attribute name as key and data as values. _var_dim : None or tuple @@ -679,13 +687,13 @@ class Nes(object): lat_bnds = self.create_single_spatial_bounds(self._lat['data'], inc_lat, spatial_nv=2) self._lat_bnds = {'data': deepcopy(lat_bnds)} - self.lat_bnds = {'data': lat_bnds[self.read_axis_limits['y_min']:self.read_axis_limits['y_max'], :]} + self.lat_bnds = {'data': lat_bnds[self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], :]} inc_lon = np.abs(np.mean(np.diff(self._lon['data']))) lon_bnds = self.create_single_spatial_bounds(self._lon['data'], inc_lon, spatial_nv=2) self._lon_bnds = {'data': deepcopy(lon_bnds)} - self.lon_bnds = {'data': lon_bnds[self.read_axis_limits['x_min']:self.read_axis_limits['x_max'], :]} + self.lon_bnds = {'data': lon_bnds[self.write_axis_limits['x_min']:self.write_axis_limits['x_max'], :]} return None diff --git a/nes/nc_projections/lcc_nes.py b/nes/nc_projections/lcc_nes.py index 0f7ff2188ea4b301d171d8aada0fe38823f5ccf8..8225c2d6bdefc7fbbe889858d4f7af7a9e85ee44 100644 --- a/nes/nc_projections/lcc_nes.py +++ b/nes/nc_projections/lcc_nes.py @@ -428,15 +428,15 @@ class LCCNes(Nes): self._lat_bnds = {} self._lat_bnds['data'] = deepcopy(lat_bnds) self.lat_bnds = {} - self.lat_bnds['data'] = lat_bnds[self.read_axis_limits['y_min']:self.read_axis_limits['y_max'], - self.read_axis_limits['x_min']:self.read_axis_limits['x_max'], + self.lat_bnds['data'] = lat_bnds[self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max'], :] self._lon_bnds = {} self._lon_bnds['data'] = deepcopy(lon_bnds) self.lon_bnds = {} - self.lon_bnds['data'] = lon_bnds[self.read_axis_limits['y_min']:self.read_axis_limits['y_max'], - self.read_axis_limits['x_min']:self.read_axis_limits['x_max'], + self.lon_bnds['data'] = lon_bnds[self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max'], :] return None diff --git a/nes/nc_projections/mercator_nes.py b/nes/nc_projections/mercator_nes.py index 01b38194e3332d99cd38b54b8a04885d707cfecf..cc02a302b3fe8ad58c72ff94ca640585cdd70c0b 100644 --- a/nes/nc_projections/mercator_nes.py +++ b/nes/nc_projections/mercator_nes.py @@ -407,15 +407,15 @@ class MercatorNes(Nes): self._lat_bnds = {} self._lat_bnds['data'] = deepcopy(lat_bnds) self.lat_bnds = {} - self.lat_bnds['data'] = lat_bnds[self.read_axis_limits['y_min']:self.read_axis_limits['y_max'], - self.read_axis_limits['x_min']:self.read_axis_limits['x_max'], + self.lat_bnds['data'] = lat_bnds[self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max'], :] self._lon_bnds = {} self._lon_bnds['data'] = deepcopy(lon_bnds) self.lon_bnds = {} - self.lon_bnds['data'] = lon_bnds[self.read_axis_limits['y_min']:self.read_axis_limits['y_max'], - self.read_axis_limits['x_min']:self.read_axis_limits['x_max'], + self.lon_bnds['data'] = lon_bnds[self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max'], :] return None diff --git a/nes/nc_projections/rotated_nes.py b/nes/nc_projections/rotated_nes.py index b7719853ef51d09aa6e125c6f48bb7312b5276a1..eccbe03320eb11485b293d6eb167e830dc14e288 100644 --- a/nes/nc_projections/rotated_nes.py +++ b/nes/nc_projections/rotated_nes.py @@ -483,15 +483,15 @@ class RotatedNes(Nes): self._lat_bnds = {} self._lat_bnds['data'] = deepcopy(lat_bnds) self.lat_bnds = {} - self.lat_bnds['data'] = lat_bnds[self.read_axis_limits['y_min']:self.read_axis_limits['y_max'], - self.read_axis_limits['x_min']:self.read_axis_limits['x_max'], + self.lat_bnds['data'] = lat_bnds[self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max'], :] self._lon_bnds = {} self._lon_bnds['data'] = deepcopy(lon_bnds) self.lon_bnds = {} - self.lon_bnds['data']= lon_bnds[self.read_axis_limits['y_min']:self.read_axis_limits['y_max'], - self.read_axis_limits['x_min']:self.read_axis_limits['x_max'], + self.lon_bnds['data']= lon_bnds[self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max'], :] return None diff --git a/tests/2.3-test_bounds.py b/tests/2.3-test_bounds.py index c2322cf19836b95b2a2d7ead065120a6e1b45392..a761858fa7e7209093bd6f474bb880152e8c202c 100644 --- a/tests/2.3-test_bounds.py +++ b/tests/2.3-test_bounds.py @@ -14,7 +14,7 @@ parallel_method = 'Y' result_path = "Times_test_2.3_bounds_{0}_{1:03d}.csv".format(parallel_method, size) result = pd.DataFrame(index=['read', 'calculate', 'write'], - columns=['2.3.1.With_bounds', '2.3.2.Without_bounds', "2.3.3.Create_new"]) + columns=['2.3.1.With_bounds', '2.3.2.Without_bounds', "2.3.3.Create_new", "2.3.4.latlon_sel_create_bnds", "2.3.5.rotated_sel_create_bnds"]) # ====================================================================================================================== # ===================================== FILE WITH EXISTING BOUNDS ==================================================== @@ -154,6 +154,125 @@ if rank == 0: print(result.loc[:, test_name]) sys.stdout.flush() + +# ====================================================================================================================== +# ================================ REGULAR LAT-LON SEL THEN CREATE BOUNDS ============================================= +# ====================================================================================================================== + +test_name = "2.3.4.latlon_sel_create_bnds" +if rank == 0: + print(test_name) + +# USE SAME GRID SETTING AS 2.3.3 +nessy_7 = create_nes(comm=None, parallel_method=parallel_method, info=True, projection='regular', + lat_orig=lat_orig, lon_orig=lon_orig, + inc_lat=inc_lat, inc_lon=inc_lon, + n_lat=n_lat, n_lon=n_lon) +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +# SEL +nessy_7.sel(lat_min=50, lat_max=60, lon_min=10, lon_max=20) + +# CREATE BOUNDS +st_time = timeit.default_timer() +nessy_7.create_spatial_bounds() +comm.Barrier() +result.loc['calculate', test_name] = timeit.default_timer() - st_time + +# EXPLORE BOUNDS +print('FROM NEW GRID - Rank', rank, '-', 'Lat bounds', nessy_7.lat_bnds) +print('FROM NEW GRID - Rank', rank, '-', 'Lon bounds', nessy_7.lon_bnds) +#print('FROM NEW GRID - Rank', rank, '-', '_Lat bounds', nessy_7._lat_bnds) +#print('FROM NEW GRID - Rank', rank, '-', '_Lon bounds', nessy_7._lon_bnds) + +# Check lon_bnds +if nessy_7.lon_bnds['data'].shape != (52, 2): + raise Exception("Wrong lon_bnds.") + +# WRITE +st_time = timeit.default_timer() +nessy_7.to_netcdf(test_name.replace(' ', '_') + "_{0:03d}.nc".format(size), info=True) +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +# REOPEN +nessy_8 = open_netcdf(test_name.replace(' ', '_') + "_{0:03d}.nc".format(size), info=True) + +# LOAD DATA AND EXPLORE BOUNDS +print('FROM NEW GRID AFTER WRITE - Rank', rank, '-', 'Lat bounds', nessy_8.lat_bnds) +print('FROM NEW GRID AFTER WRITE - Rank', rank, '-', 'Lon bounds', nessy_8.lon_bnds) +#print('FROM NEW GRID AFTER WRITE - Rank', rank, '-', '_Lat bounds', nessy_8._lat_bnds) +#print('FROM NEW GRID AFTER WRITE - Rank', rank, '-', '_Lon bounds', nessy_8._lon_bnds) + +# Check lon_bnds +if nessy_8.lon_bnds['data'].shape != (52, 2): + raise Exception("Wrong lon_bnds.") + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + + +# ====================================================================================================================== +# ================================ ROTATED SEL THEN CREATE BOUNDS ============================================= +# ====================================================================================================================== + +test_name = "2.3.5.rotated_sel_create_bnds" +if rank == 0: + print(test_name) + +# USE FILE AS 2.3.2 + +# Original path: /gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc +# Rotated grid from MONARCH +st_time = timeit.default_timer() +path_9 = "/gpfs/projects/bsc32/models/NES_tutorial_data/O3_all-000_2021080300.nc" +nessy_9 = open_netcdf(path=path_9, parallel_method=parallel_method, info=True) +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +# SEL +nessy_9.sel(lat_min=50, lat_max=60, lon_min=10, lon_max=15) + +# CREATE BOUNDS +st_time = timeit.default_timer() +nessy_9.create_spatial_bounds() +comm.Barrier() +result.loc['calculate', test_name] = timeit.default_timer() - st_time + +# EXPLORE BOUNDS +print('FILE WITHOUT EXISTING BOUNDS - Rank', rank, '-', 'Lat bounds', nessy_9.lat_bnds) +print('FILE WITHOUT EXISTING BOUNDS - Rank', rank, '-', 'Lon bounds', nessy_9.lon_bnds) + +# Check lon_bnds +if nessy_9.lon_bnds['data'].shape[0:2] != nessy_9.lon['data'].shape: + raise Exception("Wrong lon_bnds.") + +# WRITE +st_time = timeit.default_timer() +nessy_9.to_netcdf('/tmp/bounds_file_9.nc', info=True) +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +# REOPEN +nessy_10 = open_netcdf('/tmp/bounds_file_9.nc', info=True) + +# LOAD DATA AND EXPLORE BOUNDS +print('FILE WITH EXISTING BOUNDS AFTER WRITE - Rank', rank, '-', 'Lat bounds', nessy_10.lat_bnds) +print('FILE WITH EXISTING BOUNDS AFTER WRITE - Rank', rank, '-', 'Lon bounds', nessy_10.lon_bnds) + +# Check lon_bnds +if nessy_10.lon_bnds['data'].shape[0:2] != nessy_10.lon['data'].shape: + raise Exception("Wrong lon_bnds.") + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + + if rank == 0: result.to_csv(result_path) print("TEST PASSED SUCCESSFULLY!!!!!")