From 6ec0e9a1b67e8efb21b9c3b8104e8c98b17db465 Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 17 Feb 2023 10:24:22 +0100 Subject: [PATCH 1/4] BUG: bug while writing time variable after selecting only a few of them --- nes/nc_projections/default_nes.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 5520c91..abe2c28 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -2070,8 +2070,7 @@ class Nes(object): # TIMES time_var = netcdf.createVariable('time', np.float64, ('time',), zlib=self.zip_lvl > 0, complevel=self.zip_lvl) - time_var.units = 'hours since {0}'.format( - self._time[self.get_time_id(self.hours_start, first=True)].strftime('%Y-%m-%d %H:%M:%S')) + time_var.units = 'hours since {0}'.format( self._time[0].strftime('%Y-%m-%d %H:%M:%S')) time_var.standard_name = 'time' time_var.calendar = 'standard' time_var.long_name = 'time' @@ -2079,9 +2078,7 @@ class Nes(object): time_var.bounds = 'time_bnds' if self.size > 1: time_var.set_collective(True) - time_var[:] = date2num(self._time[self.get_time_id(self.hours_start, first=True): - self.get_time_id(self.hours_end, first=False)], - time_var.units, time_var.calendar) + time_var[:] = date2num(self._time[:], time_var.units, time_var.calendar) # TIME BOUNDS if self._time_bnds is not None: -- GitLab From a7e4a65caac74c3bbe83a0fbfb7e6897e2105b2a Mon Sep 17 00:00:00 2001 From: ctena Date: Fri, 17 Feb 2023 16:33:57 +0100 Subject: [PATCH 2/4] BUG: Selecting: differences over size from read & write axis --- nes/nc_projections/default_nes.py | 9 +- tests/1.3-test_selecting.py | 184 ++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 tests/1.3-test_selecting.py diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index abe2c28..ad64164 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -764,17 +764,18 @@ class Nes(object): self.lat_bnds = self._get_coordinate_values(self._lat_bnds, 'Y', bounds=True) self.lon_bnds = self._get_coordinate_values(self._lon_bnds, 'X', bounds=True) + self.filter_coordinates_selection() + # Removing complete coordinates self.write_axis_limits = self.get_write_axis_limits() - self.filter_coordinates_selection() - return None def filter_coordinates_selection(self): idx = self.get_idx_intervals() self._time = self._time[idx['idx_t_min']:idx['idx_t_max']] + self._lev['data'] = self._lev['data'][idx['idx_z_min']:idx['idx_z_max']] if len(self._lat['data'].shape) == 1: # Regular projection @@ -799,7 +800,9 @@ class Nes(object): def get_idx_intervals(self): idx = {'idx_t_min': self.get_time_id(self.hours_start, first=True), - 'idx_t_max': self.get_time_id(self.hours_end, first=False)} + 'idx_t_max': self.get_time_id(self.hours_end, first=False), + 'idx_z_min': self.first_level, + 'idx_z_max': self.last_level} # Axis Y if self.lat_min is None: diff --git a/tests/1.3-test_selecting.py b/tests/1.3-test_selecting.py new file mode 100644 index 0000000..9859812 --- /dev/null +++ b/tests/1.3-test_selecting.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +import sys +import timeit +import pandas as pd +from mpi4py import MPI +from nes import * +import os, sys +from datetime import datetime + +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +parallel_method = 'Y' +serial_write = False + +result_path = "Times_test_1.3.Selecting_{0}_{1:03d}.csv".format(parallel_method, size) + +result = pd.DataFrame(index=['read', 'calcul', 'write'], + columns=['1.3.1.LatLon', '1.3.2.Level', '1.3.3.Time', '1.3.4.Time_min', '1.3.5.Time_max']) + +# NAMEE +src_path = "/gpfs/projects/bsc32/models/NES_tutorial_data/MONARCH_d01_2022111512.nc" +var_list = ['O3'] + +# ====================================================================================================================== +# ====================================== '1.3.1.LatLon' ===================================================== +# ====================================================================================================================== +test_name = '1.3.1.LatLon' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method, balanced=True) +nessy.keep_vars(var_list) +nessy.sel(lat_min=35, lat_max=45, lon_min=-9, lon_max=5) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== 1.3.2.Level ===================================================== +# ====================================================================================================================== +test_name = '1.3.2.Level' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(lev_min=3, lev_max=5) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== 1.3.3.Time ===================================================== +# ====================================================================================================================== +test_name = '1.3.3.Time' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(time_min=datetime(year=2022, month=11, day=16, hour=0), + time_max=datetime(year=2022, month=11, day=16, hour=0)) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== '1.3.4.Time_min' ===================================================== +# ====================================================================================================================== +test_name = '1.3.4.Time_min' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(time_min=datetime(year=2022, month=11, day=16, hour=0)) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== '1.3.5.Time_max' ===================================================== +# ====================================================================================================================== +test_name = '1.3.5.Time_max' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(time_max=datetime(year=2022, month=11, day=16, hour=0)) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +if rank == 0: + result.to_csv(result_path) + print("TEST PASSED SUCCESSFULLY") -- GitLab From d050f0c929ae41fa3792be021d1fbce3f58412f5 Mon Sep 17 00:00:00 2001 From: ctena Date: Mon, 20 Feb 2023 11:41:18 +0100 Subject: [PATCH 3/4] Selecting: Fixed error while writing in serial mode --- nes/nc_projections/default_nes.py | 9 +++++++++ tests/1.3-test_selecting.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index cfcbbbe..fcb4397 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -796,6 +796,15 @@ class Nes(object): if self._lon_bnds is not None: self._lon_bnds = self._lon_bnds[idx['idx_y_min']:idx['idx_y_max'], idx['idx_x_min']:idx['idx_x_max'], :] + self.hours_start = 0 + self.hours_end = 0 + self.last_level = None + self.first_level = None + self.lat_min = None + self.lat_max = None + self.lon_max = None + self.lon_min = None + return None def get_idx_intervals(self): diff --git a/tests/1.3-test_selecting.py b/tests/1.3-test_selecting.py index 9859812..72ab997 100644 --- a/tests/1.3-test_selecting.py +++ b/tests/1.3-test_selecting.py @@ -12,7 +12,7 @@ rank = comm.Get_rank() size = comm.Get_size() parallel_method = 'Y' -serial_write = False +serial_write = True result_path = "Times_test_1.3.Selecting_{0}_{1:03d}.csv".format(parallel_method, size) -- GitLab From 95dd81054ed219f0d855bf2bff63ca6bb4799030 Mon Sep 17 00:00:00 2001 From: Alba Vilanova Date: Mon, 27 Feb 2023 05:49:08 +0100 Subject: [PATCH 4/4] Update CHANGELOG --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c7f45e..172fe41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # NES CHANGELOG ### 1.1.0 -* Release date: 2023/02/07 +* Release date: 2023/03/01 * Changes and new features: * Improve Lat-Lon to Cartesian coordinates method (used in Providentia). * Function to_shapefile() to create shapefiles from a NES object without losing data from the original grid and being able to select the time and level. @@ -9,7 +9,7 @@ * Function create_shapefile() can now be used in parallel. * Function calculate_grid_area() to calculate the area of each cell in a grid. * Function calculate_geometry_area() to calculate the area of each cell given a set of geometries. - * Function get_spatial_bounds_mesh_format() to get the lon-lat boundaries in a mesh format. (used in pcolormesh) + * Function get_spatial_bounds_mesh_format() to get the lon-lat boundaries in a mesh format (used in pcolormesh). * Bugs fixing: * Correct the dimensions of the resulting points datasets from any interpolation. * Amend the interpolation method to take into account the cases in which the distance among points equals zero. @@ -17,7 +17,10 @@ * Correct how to calculate the spatial bounds of LCC and Mercator grids: the dimensions were flipped. * Correct how to calculate the spatial bounds for all grids: use read axis limits instead of write axis limits. * Calculate centroids from coordinates in the creation of shapefiles, instead of using the geopandas function 'centroid', that raises a warning for possible errors. - * Enable selection of variables on the creation of shapefiles + * Enable selection of variables on the creation of shapefiles. + * Correct read and write parallel limits. + * Correct data type in the parallelization of points datasets. + * Correct error that appear when trying to select coordinates and write the file. ### 1.0.0 * Release date: 2022/11/24 -- GitLab