From 2295f4301093d4c0a1b768c5ece5131b97520cc6 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 8 Nov 2022 15:37:03 +0100 Subject: [PATCH 1/3] First CAMS_RA output done. Ready to be tested and used by SNES --- nes/interpolation/__init__.py | 3 + nes/nc_projections/default_nes.py | 22 +++- nes/nes_formats/__init__.py | 1 + nes/nes_formats/cams_ra_format.py | 196 ++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 nes/interpolation/__init__.py create mode 100644 nes/nes_formats/__init__.py create mode 100644 nes/nes_formats/cams_ra_format.py diff --git a/nes/interpolation/__init__.py b/nes/interpolation/__init__.py new file mode 100644 index 0000000..22c351a --- /dev/null +++ b/nes/interpolation/__init__.py @@ -0,0 +1,3 @@ +from .vertical_interpolation import add_4d_vertical_info +from .vertical_interpolation import interpolate_vertical +from .horizontal_interpolation import interpolate_horizontal diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 5628123..8f03f50 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -18,6 +18,7 @@ from copy import deepcopy, copy import datetime from ..interpolation import vertical_interpolation from ..interpolation import horizontal_interpolation +from ..nes_formats import to_netcdf_cams_ra class Nes(object): @@ -2141,8 +2142,11 @@ class Nes(object): return None + def __to_netcdf_cams_ra(self, path): + return to_netcdf_cams_ra(self, path) + def to_netcdf(self, path, compression_level=0, serial=False, info=False, - chunking=False): + chunking=False, type='NES'): """ Write the netCDF output file. @@ -2174,10 +2178,20 @@ class Nes(object): new_nc = self.copy(copy_vars=False) new_nc.set_communicator(MPI.COMM_SELF) new_nc.variables = data - new_nc.__to_netcdf_py(path) - + if type == 'NES': + new_nc.__to_netcdf_py(path) + elif type == 'CAMS_RA': + new_nc.__to_netcdf_cams_ra(path) + else: + raise ValueError( + "Unknown NetCDF type '{0}'. Use 'CAMS_RA' or 'NES'; default='NES'".format(type)) else: - self.__to_netcdf_py(path, chunking=chunking) + if type == 'NES': + self.__to_netcdf_py(path, chunking=chunking) + elif type == 'CAMS_RA': + self.__to_netcdf_cams_ra(path) + else: + raise ValueError("Unknown NetCDF type '{0}'. Use 'CAMS_RA' or 'NES'; default='NES'".format(type)) self.info = old_info diff --git a/nes/nes_formats/__init__.py b/nes/nes_formats/__init__.py new file mode 100644 index 0000000..0b0a484 --- /dev/null +++ b/nes/nes_formats/__init__.py @@ -0,0 +1 @@ +from .cams_ra_format import to_netcdf_cams_ra diff --git a/nes/nes_formats/cams_ra_format.py b/nes/nes_formats/cams_ra_format.py new file mode 100644 index 0000000..acde22f --- /dev/null +++ b/nes/nes_formats/cams_ra_format.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python + +import sys +import warnings +import numpy as np +import os +import nes +from netCDF4 import Dataset +from mpi4py import MPI +from copy import copy + + +def to_netcdf_cams_ra(self, path): + """ + Horizontal interpolation from one grid to another one. + + Parameters + ---------- + self : nes.Nes + Source projection Nes Object. + path : str + Path to the output netCDF file. + """ + if not isinstance(self, nes.LatLonNes): + raise TypeError("CAMS Re-Analysis format must have Regular Lat-Lon projection") + if '' not in path: + raise ValueError("AMS Re-Analysis path must contain '' as pattern; current: '{0}'".format(path)) + orig_path = copy(path) + for i_lev, level in enumerate(self.lev['data']): + path = orig_path.replace('', 'l{0}'.format(i_lev)) + # Open NetCDF + if self.info: + print("Rank {0:03d}: Creating {1}".format(self.rank, path)) + if self.size > 1: + netcdf = Dataset(path, format="NETCDF4", mode='w', parallel=True, comm=self.comm, info=MPI.Info()) + else: + netcdf = Dataset(path, format="NETCDF4", mode='w', parallel=False) + if self.info: + print("Rank {0:03d}: NetCDF ready to write".format(self.rank)) + self.to_dtype(data_type=np.float32) + + # Create dimensions + create_dimensions(self, netcdf) + + # Create dimension variables + create_dimension_variables(self, netcdf) + if self.info: + print("Rank {0:03d}: Dimensions done".format(self.rank)) + + # Create variables + create_variables(self, netcdf, i_lev) + + # Close NetCDF + if self.global_attrs is not None: + for att_name, att_value in self.global_attrs.items(): + netcdf.setncattr(att_name, att_value) + + netcdf.close() + + return None + + +def create_dimensions(self, netcdf): + """ + Create 'time', 'time_bnds', 'lev', 'lon' and 'lat' dimensions. + + Parameters + ---------- + self : nes.Nes + Source projection Nes Object. + netcdf : Dataset + netcdf4-python opened Dataset. + """ + + # Create time dimension + netcdf.createDimension('time', None) + + # Create lev, lon and lat dimensions + netcdf.createDimension('lon', len(self._lon['data'])) + netcdf.createDimension('lat', len(self._lat['data'])) + + return None + + +def create_dimension_variables(self, netcdf): + """ + Create the 'time', 'time_bnds', 'lev', 'lat', 'lat_bnds', 'lon' and 'lon_bnds' variables. + + Parameters + ---------- + self : nes.Nes + Source projection Nes Object. + netcdf : Dataset + netcdf4-python opened Dataset. + """ + + # TIMES + time_var = netcdf.createVariable('time', np.float64, ('time',)) + time_var.standard_name = 'time' + time_var.units = 'day as %Y%m%d.%f' + time_var.calendar = 'proleptic_gregorian' + time_var.axis = 'T' + if self.size > 1: + time_var.set_collective(True) + time_var[:] = date2num(self._time[self.get_time_id(self.hours_start, first=True): + self.get_time_id(self.hours_end, first=False)], + time_var.units, time_var.calendar) + + # LATITUDES + lat = netcdf.createVariable('lat', np.float64, ('lat',)) + lat.standard_name = 'latitude' + lat.long_name = 'latitude' + lat.units = 'degrees_north' + lat.axis = 'Y' + + if self.size > 1: + lat.set_collective(True) + lat[:] = self._lat['data'] + + # LONGITUDES + lon = netcdf.createVariable('lon', np.float64, ('lon',)) + lon.long_name = 'longitude' + lon.standard_name = 'longitude' + lon.units = 'degrees_east' + lon.axis = 'X' + if self.size > 1: + lon.set_collective(True) + lon[:] = self._lon['data'] + + return None + + +def create_variables(self, netcdf, i_lev): + """ + Create the netCDF file variables. + + Parameters + ---------- + self : nes.Nes + Source projection Nes Object. + netcdf : Dataset + netcdf4-python opened Dataset. + + """ + + for i, (var_name, var_dict) in enumerate(self.variables.items()): + if var_dict['data'] is not None: + if self.info: + print("Rank {0:03d}: Writing {1} var ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) + try: + var = netcdf.createVariable(var_name, np.float32, ('time', 'lat', 'lon',)) + + if self.info: + print("Rank {0:03d}: Var {1} created ({2}/{3})".format( + self.rank, var_name, i + 1, len(self.variables))) + if self.size > 1: + var.set_collective(True) + if self.info: + print("Rank {0:03d}: Var {1} collective ({2}/{3})".format( + self.rank, var_name, i + 1, len(self.variables))) + + if self.info: + print("Rank {0:03d}: Filling {1})".format(self.rank, var_name)) + var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], + self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = var_dict['data'][:, i_lev, :, :] + + if self.info: + print("Rank {0:03d}: Var {1} data ({2}/{3})".format( + self.rank, var_name, i + 1, len(self.variables))) + var.long_name = var_dict['long_name'] + var.units = var_dict['units'] + var.number_of_significant_digits = np.int32(3) + + if self.info: + print("Rank {0:03d}: Var {1} completed ({2}/{3})".format(self.rank, var_name, i + 1, + len(self.variables))) + except Exception as e: + print("**ERROR** an error has occurred while writing the '{0}' variable".format(var_name)) + # print("**ERROR** an error has occurredred while writing the '{0}' variable".format(var_name), + # file=sys.stderr) + raise e + else: + msg = 'WARNING!!! ' + msg += 'Variable {0} was not loaded. It will not be written.'.format(var_name) + warnings.warn(msg) + + return None + + +def date2num(time_array, time_units=None, time_calendar=None): + time_res = [] + for aux_time in time_array: + time_res.append(float(aux_time.strftime("%Y%m%d")) + (float(aux_time.strftime("%H")) / 24)) + time_res = np.array(time_res, dtype=np.float64) + return time_res -- GitLab From 86bb643b7a78bd73210d7092f93a0b16b674aa69 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 8 Nov 2022 16:02:15 +0100 Subject: [PATCH 2/3] CAMS_RA: Changed order to write items --- nes/nes_formats/cams_ra_format.py | 33 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/nes/nes_formats/cams_ra_format.py b/nes/nes_formats/cams_ra_format.py index acde22f..dd817ee 100644 --- a/nes/nes_formats/cams_ra_format.py +++ b/nes/nes_formats/cams_ra_format.py @@ -42,14 +42,14 @@ def to_netcdf_cams_ra(self, path): # Create dimensions create_dimensions(self, netcdf) + # Create variables + create_variables(self, netcdf, i_lev) + # Create dimension variables create_dimension_variables(self, netcdf) if self.info: print("Rank {0:03d}: Dimensions done".format(self.rank)) - # Create variables - create_variables(self, netcdf, i_lev) - # Close NetCDF if self.global_attrs is not None: for att_name, att_value in self.global_attrs.items(): @@ -76,8 +76,8 @@ def create_dimensions(self, netcdf): netcdf.createDimension('time', None) # Create lev, lon and lat dimensions - netcdf.createDimension('lon', len(self._lon['data'])) netcdf.createDimension('lat', len(self._lat['data'])) + netcdf.createDimension('lon', len(self._lon['data'])) return None @@ -93,19 +93,6 @@ def create_dimension_variables(self, netcdf): netcdf : Dataset netcdf4-python opened Dataset. """ - - # TIMES - time_var = netcdf.createVariable('time', np.float64, ('time',)) - time_var.standard_name = 'time' - time_var.units = 'day as %Y%m%d.%f' - time_var.calendar = 'proleptic_gregorian' - time_var.axis = 'T' - if self.size > 1: - time_var.set_collective(True) - time_var[:] = date2num(self._time[self.get_time_id(self.hours_start, first=True): - self.get_time_id(self.hours_end, first=False)], - time_var.units, time_var.calendar) - # LATITUDES lat = netcdf.createVariable('lat', np.float64, ('lat',)) lat.standard_name = 'latitude' @@ -127,6 +114,18 @@ def create_dimension_variables(self, netcdf): lon.set_collective(True) lon[:] = self._lon['data'] + # TIMES + time_var = netcdf.createVariable('time', np.float64, ('time',)) + time_var.standard_name = 'time' + time_var.units = 'day as %Y%m%d.%f' + time_var.calendar = 'proleptic_gregorian' + time_var.axis = 'T' + if self.size > 1: + time_var.set_collective(True) + time_var[:] = date2num(self._time[self.get_time_id(self.hours_start, first=True): + self.get_time_id(self.hours_end, first=False)], + time_var.units, time_var.calendar) + return None -- GitLab From f516d8d79b7bfdaaa55a2377e360ed41a8447ad8 Mon Sep 17 00:00:00 2001 From: ctena Date: Thu, 17 Nov 2022 17:54:29 +0100 Subject: [PATCH 3/3] Added compression and significant digits --- nes/nes_formats/cams_ra_format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nes/nes_formats/cams_ra_format.py b/nes/nes_formats/cams_ra_format.py index dd817ee..86bd49d 100644 --- a/nes/nes_formats/cams_ra_format.py +++ b/nes/nes_formats/cams_ra_format.py @@ -147,7 +147,8 @@ def create_variables(self, netcdf, i_lev): if self.info: print("Rank {0:03d}: Writing {1} var ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) try: - var = netcdf.createVariable(var_name, np.float32, ('time', 'lat', 'lon',)) + var = netcdf.createVariable(var_name, np.float32, ('time', 'lat', 'lon',), + zlib=True, complevel=7, least_significant_digit=3) if self.info: print("Rank {0:03d}: Var {1} created ({2}/{3})".format( -- GitLab