From 74b0031fa893fe9d8965c97c482502e3867f8f2b Mon Sep 17 00:00:00 2001 From: Alba Vilanova Cortezon Date: Thu, 5 May 2022 14:58:36 +0200 Subject: [PATCH 1/2] #1 Add time bounds --- .gitignore | 9 ++++++- nes/nc_projections/default_nes.py | 39 +++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 723ef36..695ff4b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,8 @@ -.idea \ No newline at end of file +.idea +logs +tests/basic_nes_tests_alba.py +tests/test_bash_nord3v2-alba.cmd +notebooks/.ipynb_checkpoints +.ipynb_checkpoints +nes/__pycache__ +nes/nc_projections/__pycache__ \ No newline at end of file diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 9734159..c675c6d 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -144,6 +144,7 @@ class Nes(object): self._lev = self._get_coordinate_dimension(['lev', 'level', 'lm']) self._lat = self._get_coordinate_dimension(['lat', 'latitude']) self._lon = self._get_coordinate_dimension(['lon', 'longitude']) + self._time_bnds = self.__get_time_bnds() # Axis limits self.parallel_method = parallel_method @@ -155,7 +156,8 @@ class Nes(object): self.lev = self._get_coordinate_values(self._lev, 'Z') self.lat = self._get_coordinate_values(self._lat, 'Y') self.lon = self._get_coordinate_values(self._lon, 'X') - + self.time_bnds = self._time_bnds + self.write_axis_limits = self.set_write_axis_limits() # NetCDF attributes @@ -468,6 +470,29 @@ class Nes(object): self.free_vars('time') return time + def __get_time_bnds(self): + """ + Get the NetCDF time bounds values + + Returns + ------- + time : list + List of time bounds (datetime) of the NetCDF data + """ + if self.is_xarray: + time = self.variables['time_bnds'] + else: + if self.master: + if 'time_bnds' in self.netcdf.variables.keys(): + nc_var = self.netcdf.variables['time_bnds'] + time_bnds = nc_var[:] + else: + time_bnds = None + else: + time_bnds = None + time_bnds = self.comm.bcast(time_bnds, root=0) + return time_bnds + def _get_coordinate_dimension(self, possible_names): """ Read the coordinate dimension data. @@ -719,6 +744,7 @@ class Nes(object): """ netcdf.createDimension('time', None) netcdf.createDimension('lev', len(self._lev['data'])) + netcdf.createDimension('time_nv', 2) return None @@ -738,15 +764,24 @@ class Nes(object): time_var.standard_name = "time" time_var.calendar = 'standard' time_var.long_name = "time" + if self._time_bnds is not None: + time_var.bounds = 'time_bnds' if self.size > 1: time_var.set_collective(True) time_var[:] = date2num(self._time[self.get_time_id(self.hours_start, first=True): self.get_time_id(self.hours_end, first=False)], time_var.units, calendar='standard') + # TIME BOUNDS + time_bnds_var = netcdf.createVariable('time_bnds', np.float64, ('time', 'time_nv',), zlib=self.zip_lvl, + complevel=self.zip_lvl) + if self.size > 1: + time_bnds_var.set_collective(True) + time_bnds_var[:] = self._time_bnds + # LEVELS lev = netcdf.createVariable('lev', np.float64, ('lev',), zlib=self.zip_lvl > 0, complevel=self.zip_lvl) - lev.units = Units(self._lev['units'], formatted=True).units + lev.units = self._lev['units'] lev.positive = 'up' if self.size > 1: lev.set_collective(True) -- GitLab From d70081718e84da8d4890c7bb5b80dde4d4fc59d4 Mon Sep 17 00:00:00 2001 From: Alba Vilanova Cortezon Date: Mon, 9 May 2022 13:15:24 +0200 Subject: [PATCH 2/2] #1 Add time bounds - Add warnings and improve --- nes/nc_projections/default_nes.py | 169 +++++++++++++++++++----------- 1 file changed, 105 insertions(+), 64 deletions(-) diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index c675c6d..0d9f799 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -2,6 +2,7 @@ import sys import os +from urllib.parse import _NetlocResultMixinStr import warnings import numpy as np from pandas import DataFrame, read_csv @@ -12,7 +13,7 @@ from mpi4py import MPI from cfunits import Units from numpy.ma.core import MaskError from copy import deepcopy - +import datetime class Nes(object): """ @@ -251,7 +252,7 @@ class Nes(object): def set_levels(self, levels): """ - Modify the original level values with a new ones. + Modify the original level values with new ones. Parameters ---------- @@ -262,6 +263,35 @@ class Nes(object): self.lev = deepcopy(levels) return None + def set_time_bnds(self, time_bnds): + """ + Modify the original time bounds values with new ones. + + Parameters + ---------- + time_bnds : list + List with the new time bounds information to be set + """ + correct_format = True + for time_bnd in np.array(time_bnds).flatten(): + if not isinstance(time_bnd, datetime.datetime): + print(f'{time_bnd} is not a datetime object') + correct_format = False + if correct_format: + if len(self._time) == len(time_bnds): + self._time_bnds = deepcopy(time_bnds) + self.time_bnds = deepcopy(time_bnds) + else: + msg = 'WARNING!!! ' + msg += 'The given time bounds list has a different length than the time array. Time bounds will not be set.' + warnings.warn(msg) + else: + msg = 'WARNING!!! ' + msg += 'There is at least one element in the time bounds to be set that is not a datetime object. ' + msg += 'Time bounds will not be set.' + warnings.warn(msg) + return None + def get_time_id(self, hours, first=True): """ Get the index of the corresponding time value. @@ -480,17 +510,19 @@ class Nes(object): List of time bounds (datetime) of the NetCDF data """ if self.is_xarray: - time = self.variables['time_bnds'] + time_bnds = self.variables['time_bnds'] else: if self.master: if 'time_bnds' in self.netcdf.variables.keys(): + time = self.netcdf.variables['time'] nc_var = self.netcdf.variables['time_bnds'] - time_bnds = nc_var[:] + time_bnds = num2date(nc_var[:], self.__parse_time_unit(time.units), calendar=time.calendar).tolist() else: time_bnds = None else: time_bnds = None time_bnds = self.comm.bcast(time_bnds, root=0) + self.free_vars('time_bnds') return time_bnds def _get_coordinate_dimension(self, possible_names): @@ -518,6 +550,8 @@ class Nes(object): else: nc_var = self.variables[dimension_name].copy() nc_var['data'] = self.netcdf.variables[dimension_name][:] + if nc_var['units'] in ['unitless', '-']: + nc_var['units'] = '' self.free_vars(dimension_name) return nc_var @@ -593,7 +627,7 @@ class Nes(object): # Avoiding some attributes if attrname not in ['missing_value', '_FillValue']: value = getattr(var_info, attrname) - if value == 'unitless': + if value in ['unitless', '-']: value = '' variables[var_name][attrname] = value else: @@ -744,7 +778,8 @@ class Nes(object): """ netcdf.createDimension('time', None) netcdf.createDimension('lev', len(self._lev['data'])) - netcdf.createDimension('time_nv', 2) + if self._time_bnds is not None: + netcdf.createDimension('time_nv', 2) return None @@ -773,15 +808,16 @@ class Nes(object): time_var.units, calendar='standard') # TIME BOUNDS - time_bnds_var = netcdf.createVariable('time_bnds', np.float64, ('time', 'time_nv',), zlib=self.zip_lvl, - complevel=self.zip_lvl) - if self.size > 1: - time_bnds_var.set_collective(True) - time_bnds_var[:] = self._time_bnds + if self._time_bnds is not None: + time_bnds_var = netcdf.createVariable('time_bnds', np.float64, ('time', 'time_nv',), zlib=self.zip_lvl, + complevel=self.zip_lvl) + if self.size > 1: + time_bnds_var.set_collective(True) + time_bnds_var[:] = date2num(self._time_bnds, time_var.units, calendar='standard') # LEVELS lev = netcdf.createVariable('lev', np.float64, ('lev',), zlib=self.zip_lvl > 0, complevel=self.zip_lvl) - lev.units = self._lev['units'] + lev.units = Units(self._lev['units'], formatted=True).units lev.positive = 'up' if self.size > 1: lev.set_collective(True) @@ -821,66 +857,71 @@ class Nes(object): Indicates if you want to chunk the output netCDF """ for i, (var_name, var_dict) in enumerate(self.variables.items()): - if self.print_info: - print("Rank {0:03d}: Writing {1} var ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) - try: - if not chunking: - var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, - zlib=self.zip_lvl > 0, complevel=self.zip_lvl) - else: - if self.master: - chunk_size = var_dict['data'].shape - else: - chunk_size = None - chunk_size = self.comm.bcast(chunk_size, root=0) - var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, - zlib=self.zip_lvl > 0, complevel=self.zip_lvl, chunksizes=chunk_size) + if var_dict['data'] is not None: if self.print_info: - print("Rank {0:03d}: Var {1} created ({2}/{3})".format( - self.rank, var_name, i + 1, len(self.variables))) - if self.size > 1: - var.set_collective(True) + print("Rank {0:03d}: Writing {1} var ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) + try: + if not chunking: + var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, + zlib=self.zip_lvl > 0, complevel=self.zip_lvl) + else: + if self.master: + chunk_size = var_dict['data'].shape + else: + chunk_size = None + chunk_size = self.comm.bcast(chunk_size, root=0) + var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, + zlib=self.zip_lvl > 0, complevel=self.zip_lvl, chunksizes=chunk_size) if self.print_info: - print("Rank {0:03d}: Var {1} collective ({2}/{3})".format( + print("Rank {0:03d}: Var {1} created ({2}/{3})".format( self.rank, var_name, i + 1, len(self.variables))) + if self.size > 1: + var.set_collective(True) + if self.print_info: + print("Rank {0:03d}: Var {1} collective ({2}/{3})".format( + self.rank, var_name, i + 1, len(self.variables))) + + for att_name, att_value in var_dict.items(): + if att_name == 'data': - for att_name, att_value in var_dict.items(): - if att_name == 'data': - - try: - var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], - self.write_axis_limits['z_min']:self.write_axis_limits['z_max'], - self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], - self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value - except ValueError: - var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], - 0, - self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], - self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value - # msg = "*WARNING* '{0}' variable is a 3D field. Setting it on first (0) layer.".format( - # var_name) - # warn(msg) - except IndexError: - raise IndexError("Different shapes. out_shape={0}, data_shp={1}".format( + try: var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], self.write_axis_limits['z_min']:self.write_axis_limits['z_max'], self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], - self.write_axis_limits['x_min']:self.write_axis_limits['x_max']].shape, - att_value.shape)) - if self.print_info: - print("Rank {0:03d}: Var {1} data ({2}/{3})".format(self.rank, var_name, i + 1, + self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value + except ValueError: + var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], + 0, + self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value + # msg = "*WARNING* '{0}' variable is a 3D field. Setting it on first (0) layer.".format( + # var_name) + # warn(msg) + except IndexError: + raise IndexError("Different shapes. out_shape={0}, data_shp={1}".format( + var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], + self.write_axis_limits['z_min']:self.write_axis_limits['z_max'], + self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max']].shape, + att_value.shape)) + if self.print_info: + print("Rank {0:03d}: Var {1} data ({2}/{3})".format(self.rank, var_name, i + 1, + len(self.variables))) + elif att_name not in ['chunk_size', 'var_dims', 'dimensions']: + var.setncattr(att_name, att_value) + self._set_var_crs(var) + if self.print_info: + print("Rank {0:03d}: Var {1} completed ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) - elif att_name not in ['chunk_size', 'var_dims', 'dimensions']: - var.setncattr(att_name, att_value) - self._set_var_crs(var) - if self.print_info: - print("Rank {0:03d}: Var {1} completed ({2}/{3})".format(self.rank, var_name, i + 1, - len(self.variables))) - except Exception as e: - print("**ERROR** an error has occur while writing the '{0}' variable".format(var_name)) - # print("**ERROR** an error hase occur while writing the '{0}' variable".format(var_name), - # file=sys.stderr) - raise e + except Exception as e: + print("**ERROR** an error has occurred while writing the '{0}' variable".format(var_name)) + # print("**ERROR** an error hase occurred while writing the '{0}' variable".format(var_name), + # file=sys.stderr) + raise e + else: + msg = 'WARNING!!! ' + msg += 'Variable {0} was not loaded. It will not be written.'.format(var_name) + warnings.warn(msg) return None -- GitLab