diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c7f45e6ce645ce1f2867c20cedbb73a7ac58708..172fe41d268a2124cbbab9003441fc11d80f33a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # NES CHANGELOG ### 1.1.0 -* Release date: 2023/02/07 +* Release date: 2023/03/01 * Changes and new features: * Improve Lat-Lon to Cartesian coordinates method (used in Providentia). * Function to_shapefile() to create shapefiles from a NES object without losing data from the original grid and being able to select the time and level. @@ -9,7 +9,7 @@ * Function create_shapefile() can now be used in parallel. * Function calculate_grid_area() to calculate the area of each cell in a grid. * Function calculate_geometry_area() to calculate the area of each cell given a set of geometries. - * Function get_spatial_bounds_mesh_format() to get the lon-lat boundaries in a mesh format. (used in pcolormesh) + * Function get_spatial_bounds_mesh_format() to get the lon-lat boundaries in a mesh format (used in pcolormesh). * Bugs fixing: * Correct the dimensions of the resulting points datasets from any interpolation. * Amend the interpolation method to take into account the cases in which the distance among points equals zero. @@ -17,7 +17,10 @@ * Correct how to calculate the spatial bounds of LCC and Mercator grids: the dimensions were flipped. * Correct how to calculate the spatial bounds for all grids: use read axis limits instead of write axis limits. * Calculate centroids from coordinates in the creation of shapefiles, instead of using the geopandas function 'centroid', that raises a warning for possible errors. - * Enable selection of variables on the creation of shapefiles + * Enable selection of variables on the creation of shapefiles. + * Correct read and write parallel limits. + * Correct data type in the parallelization of points datasets. + * Correct error that appear when trying to select coordinates and write the file. ### 1.0.0 * Release date: 2022/11/24 diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 1aa946a4ffc0db61f093020c97832df0f841dfcc..fcb4397c662602ced009ed4ba95d1d24d23c083b 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -764,17 +764,18 @@ class Nes(object): self.lat_bnds = self._get_coordinate_values(self._lat_bnds, 'Y', bounds=True) self.lon_bnds = self._get_coordinate_values(self._lon_bnds, 'X', bounds=True) + self.filter_coordinates_selection() + # Removing complete coordinates self.write_axis_limits = self.get_write_axis_limits() - self.filter_coordinates_selection() - return None def filter_coordinates_selection(self): idx = self.get_idx_intervals() self._time = self._time[idx['idx_t_min']:idx['idx_t_max']] + self._lev['data'] = self._lev['data'][idx['idx_z_min']:idx['idx_z_max']] if len(self._lat['data'].shape) == 1: # Regular projection @@ -795,11 +796,22 @@ class Nes(object): if self._lon_bnds is not None: self._lon_bnds = self._lon_bnds[idx['idx_y_min']:idx['idx_y_max'], idx['idx_x_min']:idx['idx_x_max'], :] + self.hours_start = 0 + self.hours_end = 0 + self.last_level = None + self.first_level = None + self.lat_min = None + self.lat_max = None + self.lon_max = None + self.lon_min = None + return None def get_idx_intervals(self): idx = {'idx_t_min': self.get_time_id(self.hours_start, first=True), - 'idx_t_max': self.get_time_id(self.hours_end, first=False)} + 'idx_t_max': self.get_time_id(self.hours_end, first=False), + 'idx_z_min': self.first_level, + 'idx_z_max': self.last_level} # Axis Y if self.lat_min is None: @@ -2070,8 +2082,7 @@ class Nes(object): # TIMES time_var = netcdf.createVariable('time', np.float64, ('time',), zlib=self.zip_lvl > 0, complevel=self.zip_lvl) - time_var.units = 'hours since {0}'.format( - self._time[self.get_time_id(self.hours_start, first=True)].strftime('%Y-%m-%d %H:%M:%S')) + time_var.units = 'hours since {0}'.format( self._time[0].strftime('%Y-%m-%d %H:%M:%S')) time_var.standard_name = 'time' time_var.calendar = 'standard' time_var.long_name = 'time' @@ -2079,9 +2090,7 @@ class Nes(object): time_var.bounds = 'time_bnds' if self.size > 1: time_var.set_collective(True) - time_var[:] = date2num(self._time[self.get_time_id(self.hours_start, first=True): - self.get_time_id(self.hours_end, first=False)], - time_var.units, time_var.calendar) + time_var[:] = date2num(self._time[:], time_var.units, time_var.calendar) # TIME BOUNDS if self._time_bnds is not None: diff --git a/tests/1.3-test_selecting.py b/tests/1.3-test_selecting.py new file mode 100644 index 0000000000000000000000000000000000000000..72ab997edfd221cc944309136e194ba57e04a40f --- /dev/null +++ b/tests/1.3-test_selecting.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +import sys +import timeit +import pandas as pd +from mpi4py import MPI +from nes import * +import os, sys +from datetime import datetime + +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +parallel_method = 'Y' +serial_write = True + +result_path = "Times_test_1.3.Selecting_{0}_{1:03d}.csv".format(parallel_method, size) + +result = pd.DataFrame(index=['read', 'calcul', 'write'], + columns=['1.3.1.LatLon', '1.3.2.Level', '1.3.3.Time', '1.3.4.Time_min', '1.3.5.Time_max']) + +# NAMEE +src_path = "/gpfs/projects/bsc32/models/NES_tutorial_data/MONARCH_d01_2022111512.nc" +var_list = ['O3'] + +# ====================================================================================================================== +# ====================================== '1.3.1.LatLon' ===================================================== +# ====================================================================================================================== +test_name = '1.3.1.LatLon' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method, balanced=True) +nessy.keep_vars(var_list) +nessy.sel(lat_min=35, lat_max=45, lon_min=-9, lon_max=5) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== 1.3.2.Level ===================================================== +# ====================================================================================================================== +test_name = '1.3.2.Level' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(lev_min=3, lev_max=5) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== 1.3.3.Time ===================================================== +# ====================================================================================================================== +test_name = '1.3.3.Time' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(time_min=datetime(year=2022, month=11, day=16, hour=0), + time_max=datetime(year=2022, month=11, day=16, hour=0)) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== '1.3.4.Time_min' ===================================================== +# ====================================================================================================================== +test_name = '1.3.4.Time_min' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(time_min=datetime(year=2022, month=11, day=16, hour=0)) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +# ====================================================================================================================== +# ====================================== '1.3.5.Time_max' ===================================================== +# ====================================================================================================================== +test_name = '1.3.5.Time_max' + +if rank == 0: + print(test_name) + +st_time = timeit.default_timer() + +# Source data +nessy = open_netcdf(src_path, parallel_method=parallel_method) +nessy.keep_vars(var_list) +nessy.sel(time_max=datetime(year=2022, month=11, day=16, hour=0)) + +nessy.load() + +comm.Barrier() +result.loc['read', test_name] = timeit.default_timer() - st_time + +st_time = timeit.default_timer() +nessy.to_netcdf(test_name.replace(' ', '_') + "{0:03d}.nc".format(size), serial=serial_write) + +comm.Barrier() +result.loc['write', test_name] = timeit.default_timer() - st_time + +comm.Barrier() +if rank == 0: + print(result.loc[:, test_name]) +sys.stdout.flush() + +if rank == 0: + result.to_csv(result_path) + print("TEST PASSED SUCCESSFULLY")