From 513cf4e4bd7cfe02b2dded0e9ebb84669d135edd Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 3 Jan 2024 15:41:56 +0100 Subject: [PATCH 1/3] - Removed xarray - Starting open_raster() --- nes/__init__.py | 1 + nes/create_nes.py | 12 +- nes/load_nes.py | 65 ++-- nes/methods/vertical_interpolation.py | 1 - nes/nc_projections/__init__.py | 1 + nes/nc_projections/default_nes.py | 293 +++++++------------ nes/nc_projections/latlon_nes.py | 12 +- nes/nc_projections/lcc_nes.py | 12 +- nes/nc_projections/mercator_nes.py | 12 +- nes/nc_projections/points_nes.py | 12 +- nes/nc_projections/points_nes_ghost.py | 12 +- nes/nc_projections/points_nes_providentia.py | 15 +- nes/nc_projections/raster_nes.py | 13 + nes/nc_projections/rotated_nes.py | 12 +- nes/nc_projections/rotated_nested_nes.py | 7 +- requirements.txt | 4 +- 16 files changed, 214 insertions(+), 270 deletions(-) create mode 100644 nes/nc_projections/raster_nes.py diff --git a/nes/__init__.py b/nes/__init__.py index 884723b..86a702f 100644 --- a/nes/__init__.py +++ b/nes/__init__.py @@ -2,6 +2,7 @@ __date__ = "2023-06-22" __version__ = "1.1.3" from .load_nes import open_netcdf, concatenate_netcdfs +from .load_nes import open_raster from .create_nes import create_nes, from_shapefile from .nc_projections import * from .methods.cell_measures import calculate_geometry_area diff --git a/nes/create_nes.py b/nes/create_nes.py index 7444bb5..98f81f0 100644 --- a/nes/create_nes.py +++ b/nes/create_nes.py @@ -92,32 +92,32 @@ def create_nes(comm=None, info=False, projection=None, parallel_method='Y', bala parallel_method = 'X' elif parallel_method == 'T': raise NotImplementedError("Parallel method T not implemented yet") - nessy = PointsNes(comm=comm, dataset=None, xarray=False, info=info, parallel_method=parallel_method, + nessy = PointsNes(comm=comm, dataset=None, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, balanced=balanced, create_nes=True, times=times, **kwargs) elif projection in ['regular', 'global']: - nessy = LatLonNes(comm=comm, dataset=None, xarray=False, info=info, parallel_method=parallel_method, + nessy = LatLonNes(comm=comm, dataset=None, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, balanced=balanced, create_nes=True, times=times, **kwargs) elif projection == 'rotated': - nessy = RotatedNes(comm=comm, dataset=None, xarray=False, info=info, parallel_method=parallel_method, + nessy = RotatedNes(comm=comm, dataset=None, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, balanced=balanced, create_nes=True, times=times, **kwargs) elif projection == 'rotated-nested': - nessy = RotatedNestedNes(comm=comm, dataset=None, xarray=False, info=info, parallel_method=parallel_method, + nessy = RotatedNestedNes(comm=comm, dataset=None, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, balanced=balanced, create_nes=True, times=times, **kwargs) elif projection == 'lcc': - nessy = LCCNes(comm=comm, dataset=None, xarray=False, info=info, parallel_method=parallel_method, + nessy = LCCNes(comm=comm, dataset=None, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, balanced=balanced, create_nes=True, times=times, **kwargs) elif projection == 'mercator': - nessy = MercatorNes(comm=comm, dataset=None, xarray=False, info=info, parallel_method=parallel_method, + nessy = MercatorNes(comm=comm, dataset=None, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, balanced=balanced, create_nes=True, times=times, **kwargs) diff --git a/nes/load_nes.py b/nes/load_nes.py index dd08add..0fb271e 100644 --- a/nes/load_nes.py +++ b/nes/load_nes.py @@ -12,7 +12,7 @@ DIM_VAR_NAMES = ['lat', 'latitude', 'lat_bnds', 'lon', 'longitude', 'lon_bnds', 'cell_area', 'crs', 'rotated_pole', 'x', 'y', 'rlat', 'rlon', 'Lambert_conformal', 'mercator'] -def open_netcdf(path, comm=None, xarray=False, info=False, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, +def open_netcdf(path, comm=None, info=False, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, balanced=False): """ Open a netCDF file. @@ -23,8 +23,6 @@ def open_netcdf(path, comm=None, xarray=False, info=False, parallel_method='Y', Path to the NetCDF file to read. comm : MPI.COMM MPI communicator to use in that netCDF. Default: MPI.COMM_WORLD. - xarray : bool - (Not working) Indicates if you want to use xarray. Default: False. info : bool Indicates if you want to print (stdout) the reading/writing steps. avoid_first_hours : int @@ -54,19 +52,17 @@ def open_netcdf(path, comm=None, xarray=False, info=False, parallel_method='Y', if not os.path.exists(path): raise FileNotFoundError(path) - if xarray: - dataset = None - else: - dataset = Dataset(path, format="NETCDF4", mode='r', parallel=False) - # Parallel is not needed for reading - # if comm.Get_size() == 1: - # dataset = Dataset(path, format="NETCDF4", mode='r', parallel=False) - # else: - # dataset = Dataset(path, format="NETCDF4", mode='r', parallel=True, comm=comm, info=MPI.Info()) + + dataset = Dataset(path, format="NETCDF4", mode='r', parallel=False) + # Parallel is not needed for reading + # if comm.Get_size() == 1: + # dataset = Dataset(path, format="NETCDF4", mode='r', parallel=False) + # else: + # dataset = Dataset(path, format="NETCDF4", mode='r', parallel=True, comm=comm, info=MPI.Info()) if __is_rotated(dataset): # Rotated grids - nessy = RotatedNes(comm=comm, dataset=dataset, xarray=xarray, info=info, parallel_method=parallel_method, + nessy = RotatedNes(comm=comm, dataset=dataset, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=False, balanced=balanced,) elif __is_points(dataset): @@ -76,35 +72,35 @@ def open_netcdf(path, comm=None, xarray=False, info=False, parallel_method='Y', parallel_method = 'X' if __is_points_ghost(dataset): # Points - GHOST - nessy = PointsNesGHOST(comm=comm, dataset=dataset, xarray=xarray, info=info, + nessy = PointsNesGHOST(comm=comm, dataset=dataset, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=False, balanced=balanced,) elif __is_points_providentia(dataset): # Points - Providentia - nessy = PointsNesProvidentia(comm=comm, dataset=dataset, xarray=xarray, info=info, + nessy = PointsNesProvidentia(comm=comm, dataset=dataset, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=False, balanced=balanced,) else: # Points - non-GHOST - nessy = PointsNes(comm=comm, dataset=dataset, xarray=xarray, info=info, parallel_method=parallel_method, + nessy = PointsNes(comm=comm, dataset=dataset, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=False, balanced=balanced,) elif __is_lcc(dataset): # Lambert conformal conic grids - nessy = LCCNes(comm=comm, dataset=dataset, xarray=xarray, info=info, parallel_method=parallel_method, + nessy = LCCNes(comm=comm, dataset=dataset, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=False, balanced=balanced,) elif __is_mercator(dataset): # Mercator grids - nessy = MercatorNes(comm=comm, dataset=dataset, xarray=xarray, info=info, parallel_method=parallel_method, + nessy = MercatorNes(comm=comm, dataset=dataset, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=False, balanced=balanced,) else: # Regular grids - nessy = LatLonNes(comm=comm, dataset=dataset, xarray=xarray, info=info, parallel_method=parallel_method, + nessy = LatLonNes(comm=comm, dataset=dataset, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=False, balanced=balanced,) @@ -326,3 +322,34 @@ def concatenate_netcdfs(nessy_list, comm=None, info=False, parallel_method='Y', nessy_first.concatenate(aux_nessy) return nessy_first + + +def open_raster(path, comm=None, info=False): + """ + Open a Raster (TIF) file + + Parameters + ---------- + path : str + Path to the Raster (TIFF) file to read. + comm : MPI.COMM + MPI communicator to use in that netCDF. Default: MPI.COMM_WORLD. + info : bool + Indicates if you want to print (stdout) the reading/writing steps. + + Returns + ------- + RasterNes + Raster NES object + """ + if comm is None: + comm = MPI.COMM_WORLD + else: + comm = comm + + if not os.path.exists(path): + raise FileNotFoundError(path) + + nessy = RasterNes(path=path, comm=comm, info=info) + + return nessy diff --git a/nes/methods/vertical_interpolation.py b/nes/methods/vertical_interpolation.py index 5a980eb..995207b 100644 --- a/nes/methods/vertical_interpolation.py +++ b/nes/methods/vertical_interpolation.py @@ -194,7 +194,6 @@ def interpolate_vertical(self, new_levels, new_src_vertical=None, kind='linear', # Remove original file information self.__ini_path = None - self.dataset = None self.netcdf = None return self diff --git a/nes/nc_projections/__init__.py b/nes/nc_projections/__init__.py index d4c4b9f..f5c287b 100644 --- a/nes/nc_projections/__init__.py +++ b/nes/nc_projections/__init__.py @@ -7,3 +7,4 @@ from .points_nes_ghost import PointsNesGHOST from .points_nes_providentia import PointsNesProvidentia from .lcc_nes import LCCNes from .mercator_nes import MercatorNes +from .raster_nes import RasterNes diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 55cd1a4..12efaff 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -5,7 +5,6 @@ import gc import warnings import numpy as np import pandas as pd -from xarray import open_dataset from netCDF4 import Dataset, num2date, date2num, stringtochar from mpi4py import MPI from cfunits import Units @@ -35,8 +34,6 @@ class Nes(object): Size of the communicator. info : bool Indicates if you want to print reading/writing info. - is_xarray : bool - (Not working) Indicates if you want to use xarray as default. __ini_path : str Path to the original file to read when open_netcdf is called. hours_start : int @@ -91,7 +88,7 @@ class Nes(object): Dictionary with the projection information. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -105,10 +102,8 @@ class Nes(object): Path to the NetCDF to initialize the object. info: bool Indicates if you want to get reading/writing info. - dataset: Dataset + dataset: Dataset or None NetCDF4-python Dataset to initialize the class. - xarray: bool - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default over Y axis accepted values: ['X', 'Y', 'T']. @@ -140,7 +135,6 @@ class Nes(object): # General info self.info = info - self.is_xarray = xarray self.__ini_path = path self.shapefile = None @@ -171,7 +165,6 @@ class Nes(object): if create_nes: self.netcdf = None - self.dataset = None # Set string length self.strlen = None @@ -208,21 +201,10 @@ class Nes(object): self.global_attrs = self.__get_global_attributes(create_nes) else: - if dataset is not None: - if self.is_xarray: - self.dataset = dataset - self.netcdf = None - else: - self.dataset = None - self.netcdf = dataset + self.netcdf = dataset elif self.__ini_path is not None: - if self.is_xarray: - self.dataset = self.__open_dataset() - self.netcdf = None - else: - self.dataset = None - self.netcdf = self.__open_netcdf4() + self.open() # Get string length self.strlen = self._get_strlen() @@ -286,7 +268,7 @@ class Nes(object): self.first_level = None @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -302,8 +284,6 @@ class Nes(object): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default over Y axis accepted values: ['X', 'Y', 'T']. @@ -324,7 +304,7 @@ class Nes(object): List of times to substitute the current ones while creation. """ - new = Nes(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, parallel_method=parallel_method, + new = Nes(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, **kwargs) @@ -745,16 +725,12 @@ class Nes(object): if isinstance(var_list, str): var_list = [var_list] - if self.is_xarray: - self.dataset = self.dataset.drop_vars(var_list) - self.variables = self._get_lazy_variables() - else: - if self.variables is not None: - for var_name in var_list: - if var_name in self.variables: - if 'data' in self.variables[var_name].keys(): - del self.variables[var_name]['data'] - del self.variables[var_name] + if self.variables is not None: + for var_name in var_list: + if var_name in self.variables: + if 'data' in self.variables[var_name].keys(): + del self.variables[var_name]['data'] + del self.variables[var_name] gc.collect() return None @@ -1457,37 +1433,10 @@ class Nes(object): Open the NetCDF. """ - if self.is_xarray: - self.dataset = self.__open_dataset() - self.netcdf = None - else: - self.dataset = None - self.netcdf = self.__open_netcdf4() + self.netcdf = self.__open_netcdf4() return None - def __open_dataset(self): - """ - Open the NetCDF with xarray. - - Returns - ------- - dataset : xr.Dataset - Open dataset. - """ - - if self.master: - warnings.filterwarnings('ignore') # Disabling warnings while reading MONARCH original file - dataset = open_dataset(self.__ini_path, decode_coords='all') - warnings.filterwarnings('default') # Re-activating warnings - else: - dataset = None - - dataset = self.comm.bcast(dataset, root=0) - self.dataset = dataset - - return dataset - def __open_netcdf4(self, mode='r'): """ Open the NetCDF with netcdf4-python. @@ -1660,23 +1609,20 @@ class Nes(object): List of times (datetime.datetime) of the NetCDF data. """ - if self.is_xarray: - time = self.variables['time'] + if self.master: + nc_var = self.netcdf.variables['time'] + time_data, units, calendar = self.__parse_time(nc_var) + # Extracting time resolution depending on the units + self._time_resolution = self.__get_time_resolution_from_units(units) + # Checking if it is a climatology dataset + if hasattr(nc_var, 'climatology'): + self._climatology = True + self._climatology_var_name = nc_var.climatology + time = num2date(time_data, units, calendar=calendar) + time = [aux.replace(second=0, microsecond=0) for aux in time] else: - if self.master: - nc_var = self.netcdf.variables['time'] - time_data, units, calendar = self.__parse_time(nc_var) - # Extracting time resolution depending on the units - self._time_resolution = self.__get_time_resolution_from_units(units) - # Checking if it is a climatology dataset - if hasattr(nc_var, 'climatology'): - self._climatology = True - self._climatology_var_name = nc_var.climatology - time = num2date(time_data, units, calendar=calendar) - time = [aux.replace(second=0, microsecond=0) for aux in time] - else: - time = None - time = self.comm.bcast(time, root=0) + time = None + time = self.comm.bcast(time, root=0) self.free_vars('time') return time @@ -1736,28 +1682,24 @@ class Nes(object): Longitude bounds of the NetCDF data. """ - if self.is_xarray: - lat_bnds = self.variables['lat_bnds'] - lon_bnds = self.variables['lon_bnds'] - else: - if self.master: - if not create_nes: - if 'lat_bnds' in self.netcdf.variables.keys(): - lat_bnds = {'data': self._unmask_array(self.netcdf.variables['lat_bnds'][:])} - else: - lat_bnds = None - if 'lon_bnds' in self.netcdf.variables.keys(): - lon_bnds = {'data': self._unmask_array(self.netcdf.variables['lon_bnds'][:])} - else: - lon_bnds = None + if self.master: + if not create_nes: + if 'lat_bnds' in self.netcdf.variables.keys(): + lat_bnds = {'data': self._unmask_array(self.netcdf.variables['lat_bnds'][:])} else: lat_bnds = None + if 'lon_bnds' in self.netcdf.variables.keys(): + lon_bnds = {'data': self._unmask_array(self.netcdf.variables['lon_bnds'][:])} + else: lon_bnds = None else: lat_bnds = None lon_bnds = None - lat_bnds = self.comm.bcast(lat_bnds, root=0) - lon_bnds = self.comm.bcast(lon_bnds, root=0) + else: + lat_bnds = None + lon_bnds = None + lat_bnds = self.comm.bcast(lat_bnds, root=0) + lon_bnds = self.comm.bcast(lon_bnds, root=0) self.free_vars(['lat_bnds', 'lon_bnds']) @@ -1812,12 +1754,10 @@ class Nes(object): try: dimension_name = set(possible_names).intersection(set(self.variables.keys())).pop() - if self.is_xarray: - nc_var = self.dataset[dimension_name] - else: - nc_var = self.variables[dimension_name].copy() - nc_var['data'] = self.netcdf.variables[dimension_name][:] - if hasattr(nc_var, 'units'): + + nc_var = self.variables[dimension_name].copy() + nc_var['data'] = self.netcdf.variables[dimension_name][:] + if hasattr(nc_var, 'units'): if nc_var['units'] in ['unitless', '-']: nc_var['units'] = '' self.free_vars(dimension_name) @@ -1938,32 +1878,29 @@ class Nes(object): ...} """ - if self.is_xarray: - variables = self.dataset.variables + if self.master: + variables = {} + # Initialise data + for var_name, var_info in self.netcdf.variables.items(): + variables[var_name] = {} + variables[var_name]['data'] = None + variables[var_name]['dimensions'] = var_info.dimensions + variables[var_name]['dtype'] = var_info.dtype + if variables[var_name]['dtype'] in [str, np.object]: + if self.strlen is None: + self.set_strlen() + variables[var_name]['dtype'] = str + + # Avoid some attributes + for attrname in var_info.ncattrs(): + if attrname not in ['missing_value', '_FillValue']: + value = getattr(var_info, attrname) + if value in ['unitless', '-']: + value = '' + variables[var_name][attrname] = value else: - if self.master: - variables = {} - # Initialise data - for var_name, var_info in self.netcdf.variables.items(): - variables[var_name] = {} - variables[var_name]['data'] = None - variables[var_name]['dimensions'] = var_info.dimensions - variables[var_name]['dtype'] = var_info.dtype - if variables[var_name]['dtype'] in [str, np.object]: - if self.strlen is None: - self.set_strlen() - variables[var_name]['dtype'] = str - - # Avoid some attributes - for attrname in var_info.ncattrs(): - if attrname not in ['missing_value', '_FillValue']: - value = getattr(var_info, attrname) - if value in ['unitless', '-']: - value = '' - variables[var_name][attrname] = value - else: - variables = None - variables = self.comm.bcast(variables, root=0) + variables = None + variables = self.comm.bcast(variables, root=0) return variables @@ -2057,7 +1994,7 @@ class Nes(object): List (or single string) of the variables to be loaded. """ - if (self.__ini_path is None) and (self.dataset is None) and (self.netcdf is None): + if (self.__ini_path is None) and (self.netcdf is None): raise RuntimeError('Only data from existing files can be loaded.') if self.netcdf is None: @@ -2154,7 +2091,6 @@ class Nes(object): if isinstance(aux_nessy, str): aux_nessy = self.new(path=aux_nessy, comm=self.comm, parallel_method=self.parallel_method, - xarray=self.is_xarray, avoid_first_hours=self.hours_start, avoid_last_hours=self.hours_end, first_level=self.first_level, last_level=self.last_level) new = True @@ -2192,12 +2128,10 @@ class Nes(object): """ gl_attrs = {} - if self.is_xarray: - gl_attrs = self.dataset.attrs - else: - if not create_nes: - for attrname in self.netcdf.ncattrs(): - gl_attrs[attrname] = getattr(self.netcdf, attrname) + + if not create_nes: + for attrname in self.netcdf.ncattrs(): + gl_attrs[attrname] = getattr(self.netcdf, attrname) return gl_attrs @@ -2891,7 +2825,6 @@ class Nes(object): def __to_netcdf_cams_ra(self, path): return to_netcdf_cams_ra(self, path) - def to_netcdf(self, path, compression_level=0, serial=False, info=False, chunking=False, type='NES', keep_open=False): """ @@ -2919,56 +2852,54 @@ class Nes(object): self.info = info self.serial_nc = None self.zip_lvl = compression_level - if self.is_xarray: - raise NotImplementedError("Writing with xarray not implemented") - else: - # if serial: - if serial and self.size > 1: - try: - data = self._gather_data(self.variables) - except KeyError: - data = self.__gather_data_py_object(self.variables) - try: - c_measures = self._gather_data(self.cell_measures) - except KeyError: - c_measures = self.__gather_data_py_object(self.cell_measures) - if self.master: - new_nc = self.copy(copy_vars=False) - new_nc.set_communicator(MPI.COMM_SELF) - new_nc.variables = data - new_nc.cell_measures = c_measures - if type in ['NES', 'DEFAULT']: - new_nc.__to_netcdf_py(path, keep_open=keep_open) - elif type == 'CAMS_RA': - new_nc.__to_netcdf_cams_ra(path) - elif type == 'MONARCH': - to_netcdf_monarch(new_nc, path, chunking=chunking, keep_open=keep_open) - elif type == 'CMAQ': - to_netcdf_cmaq(new_nc, path, keep_open=keep_open) - elif type == 'WRF_CHEM': - to_netcdf_wrf_chem(new_nc, path, keep_open=keep_open) - else: - msg = "Unknown NetCDF type '{0}'. ".format(nc_type) - msg += "Use CAMS_RA, MONARCH or NES (or DEFAULT)" - raise ValueError(msg) - self.serial_nc = new_nc - else: - self.serial_nc = True - else: + + # if serial: + if serial and self.size > 1: + try: + data = self._gather_data(self.variables) + except KeyError: + data = self.__gather_data_py_object(self.variables) + try: + c_measures = self._gather_data(self.cell_measures) + except KeyError: + c_measures = self.__gather_data_py_object(self.cell_measures) + if self.master: + new_nc = self.copy(copy_vars=False) + new_nc.set_communicator(MPI.COMM_SELF) + new_nc.variables = data + new_nc.cell_measures = c_measures if type in ['NES', 'DEFAULT']: - self.__to_netcdf_py(path, chunking=chunking, keep_open=keep_open) - elif nc_type == 'CAMS_RA': - self.__to_netcdf_cams_ra(path) - elif nc_type == 'MONARCH': - to_netcdf_monarch(self, path, chunking=chunking, keep_open=keep_open) - elif nc_type == 'CMAQ': - to_netcdf_cmaq(self, path, keep_open=keep_open) - elif nc_type == 'WRF_CHEM': - to_netcdf_wrf_chem(self, path, keep_open=keep_open) + new_nc.__to_netcdf_py(path, keep_open=keep_open) + elif type == 'CAMS_RA': + new_nc.__to_netcdf_cams_ra(path) + elif type == 'MONARCH': + to_netcdf_monarch(new_nc, path, chunking=chunking, keep_open=keep_open) + elif type == 'CMAQ': + to_netcdf_cmaq(new_nc, path, keep_open=keep_open) + elif type == 'WRF_CHEM': + to_netcdf_wrf_chem(new_nc, path, keep_open=keep_open) else: msg = "Unknown NetCDF type '{0}'. ".format(nc_type) msg += "Use CAMS_RA, MONARCH or NES (or DEFAULT)" raise ValueError(msg) + self.serial_nc = new_nc + else: + self.serial_nc = True + else: + if type in ['NES', 'DEFAULT']: + self.__to_netcdf_py(path, chunking=chunking, keep_open=keep_open) + elif nc_type == 'CAMS_RA': + self.__to_netcdf_cams_ra(path) + elif nc_type == 'MONARCH': + to_netcdf_monarch(self, path, chunking=chunking, keep_open=keep_open) + elif nc_type == 'CMAQ': + to_netcdf_cmaq(self, path, keep_open=keep_open) + elif nc_type == 'WRF_CHEM': + to_netcdf_wrf_chem(self, path, keep_open=keep_open) + else: + msg = "Unknown NetCDF type '{0}'. ".format(nc_type) + msg += "Use CAMS_RA, MONARCH or NES (or DEFAULT)" + raise ValueError(msg) self.info = old_info diff --git a/nes/nc_projections/latlon_nes.py b/nes/nc_projections/latlon_nes.py index 4b38959..05917fc 100644 --- a/nes/nc_projections/latlon_nes.py +++ b/nes/nc_projections/latlon_nes.py @@ -20,7 +20,7 @@ class LatLonNes(Nes): Tuple with the name of the dimensions of the Longitude values. ('lon',) for a regular latitude-longitude projection. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -36,8 +36,6 @@ class LatLonNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -59,7 +57,7 @@ class LatLonNes(Nes): """ super(LatLonNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, - xarray=xarray, parallel_method=parallel_method, balanced=balanced, + parallel_method=parallel_method, balanced=balanced, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, times=times, **kwargs) @@ -79,7 +77,7 @@ class LatLonNes(Nes): self.free_vars('crs') @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -95,8 +93,6 @@ class LatLonNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -117,7 +113,7 @@ class LatLonNes(Nes): List of times to substitute the current ones while creation. """ - new = LatLonNes(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, + new = LatLonNes(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, **kwargs) diff --git a/nes/nc_projections/lcc_nes.py b/nes/nc_projections/lcc_nes.py index 0f7ff21..f697891 100644 --- a/nes/nc_projections/lcc_nes.py +++ b/nes/nc_projections/lcc_nes.py @@ -35,7 +35,7 @@ class LCCNes(Nes): Tuple with the name of the dimensions of the Longitude values. ('y', 'x') for a LCC projection. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -51,8 +51,6 @@ class LCCNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -74,7 +72,7 @@ class LCCNes(Nes): """ super(LCCNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, - xarray=xarray, parallel_method=parallel_method, balanced=balanced, + parallel_method=parallel_method, balanced=balanced, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, times=times, **kwargs) @@ -102,7 +100,7 @@ class LCCNes(Nes): self.free_vars('crs') @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -118,8 +116,6 @@ class LCCNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -140,7 +136,7 @@ class LCCNes(Nes): List of times to substitute the current ones while creation. """ - new = LCCNes(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, + new = LCCNes(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, **kwargs) diff --git a/nes/nc_projections/mercator_nes.py b/nes/nc_projections/mercator_nes.py index 01b3819..52b8ab9 100644 --- a/nes/nc_projections/mercator_nes.py +++ b/nes/nc_projections/mercator_nes.py @@ -35,7 +35,7 @@ class MercatorNes(Nes): Tuple with the name of the dimensions of the Longitude values. ('y', 'x') for a Mercator projection. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -51,8 +51,6 @@ class MercatorNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -75,7 +73,7 @@ class MercatorNes(Nes): """ super(MercatorNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, - xarray=xarray, parallel_method=parallel_method, balanced=balanced, + parallel_method=parallel_method, balanced=balanced, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, times=times, **kwargs) @@ -103,7 +101,7 @@ class MercatorNes(Nes): self.free_vars('crs') @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -119,8 +117,6 @@ class MercatorNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -141,7 +137,7 @@ class MercatorNes(Nes): List of times to substitute the current ones while creation. """ - new = MercatorNes(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, + new = MercatorNes(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, **kwargs) diff --git a/nes/nc_projections/points_nes.py b/nes/nc_projections/points_nes.py index ee54184..bca62c0 100644 --- a/nes/nc_projections/points_nes.py +++ b/nes/nc_projections/points_nes.py @@ -28,7 +28,7 @@ class PointsNes(Nes): Tuple with the name of the dimensions of the station values. ('station',) for a points grid. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='X', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='X', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -44,8 +44,6 @@ class PointsNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset, None NetCDF4-python Dataset to initialize the class. - xarray: bool - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'X'. accepted values: ['X', 'T']. @@ -69,7 +67,7 @@ class PointsNes(Nes): """ super(PointsNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, - xarray=xarray, parallel_method=parallel_method, + parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, times=times, **kwargs) @@ -93,7 +91,7 @@ class PointsNes(Nes): self._lon_dim = ('station',) @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='X', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='X', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -109,8 +107,6 @@ class PointsNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'X'. accepted values: ['X', 'T']. @@ -131,7 +127,7 @@ class PointsNes(Nes): List of times to substitute the current ones while creation. """ - new = PointsNes(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, + new = PointsNes(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, **kwargs) diff --git a/nes/nc_projections/points_nes_ghost.py b/nes/nc_projections/points_nes_ghost.py index 09c5a44..2acdd81 100644 --- a/nes/nc_projections/points_nes_ghost.py +++ b/nes/nc_projections/points_nes_ghost.py @@ -25,7 +25,7 @@ class PointsNesGHOST(PointsNes): Data flags (given by data provider) dictionary with the portion of 'data' corresponding to the rank values. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='X', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='X', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -41,8 +41,6 @@ class PointsNesGHOST(PointsNes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'X'. Accepted values: ['X']. @@ -64,7 +62,7 @@ class PointsNesGHOST(PointsNes): """ super(PointsNesGHOST, self).__init__(comm=comm, path=path, info=info, dataset=dataset, - xarray=xarray, parallel_method=parallel_method, + parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, times=times, **kwargs) @@ -78,7 +76,7 @@ class PointsNesGHOST(PointsNes): self.qa = self._get_coordinate_values(self._qa, 'X') @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='X', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='X', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -94,8 +92,6 @@ class PointsNesGHOST(PointsNes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'X'. Accepted values: ['X']. @@ -116,7 +112,7 @@ class PointsNesGHOST(PointsNes): List of times to substitute the current ones while creation. """ - new = PointsNesGHOST(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, + new = PointsNesGHOST(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, **kwargs) diff --git a/nes/nc_projections/points_nes_providentia.py b/nes/nc_projections/points_nes_providentia.py index 533e4a8..b157cb3 100644 --- a/nes/nc_projections/points_nes_providentia.py +++ b/nes/nc_projections/points_nes_providentia.py @@ -34,9 +34,10 @@ class PointsNesProvidentia(PointsNes): grid_edge_lat : dict Grid edge latitudes dictionary with the portion of 'data' corresponding to the rank values. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='X', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='X', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, - balanced=False, times=None, model_centre_lon=None, model_centre_lat=None, grid_edge_lon=None, grid_edge_lat=None, + balanced=False, times=None, model_centre_lon=None, model_centre_lat=None, grid_edge_lon=None, + grid_edge_lat=None, **kwargs): """ Initialize the PointsNesProvidentia class @@ -51,8 +52,6 @@ class PointsNesProvidentia(PointsNes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'X'. Accepted values: ['X']. @@ -82,7 +81,7 @@ class PointsNesProvidentia(PointsNes): """ super(PointsNesProvidentia, self).__init__(comm=comm, path=path, info=info, dataset=dataset, - xarray=xarray, parallel_method=parallel_method, + parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, @@ -108,7 +107,7 @@ class PointsNesProvidentia(PointsNes): self.grid_edge_lat = self._get_coordinate_values(self._grid_edge_lat, '') @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='X', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='X', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, model_centre_lon=None, model_centre_lat=None, grid_edge_lon=None, grid_edge_lat=None, @@ -126,8 +125,6 @@ class PointsNesProvidentia(PointsNes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'X'. Accepted values: ['X']. @@ -156,7 +153,7 @@ class PointsNesProvidentia(PointsNes): Grid edge latitudes dictionary with the portion of 'data' corresponding to the rank values. """ - new = PointsNesProvidentia(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, + new = PointsNesProvidentia(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, diff --git a/nes/nc_projections/raster_nes.py b/nes/nc_projections/raster_nes.py new file mode 100644 index 0000000..51d5e7d --- /dev/null +++ b/nes/nc_projections/raster_nes.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import numpy as np +from pyproj import Proj +from .default_nes import Nes + + +class RasterNes(Nes): + + def __init__(self, comm=None, path=None, info=False, **kwargs): + super().__init__(comm=comm, path=path, info=info, dataset=None, parallel_method='Y', + balanced=False, avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, + create_nes=False, times=None, **kwargs) diff --git a/nes/nc_projections/rotated_nes.py b/nes/nc_projections/rotated_nes.py index b771985..125736d 100644 --- a/nes/nc_projections/rotated_nes.py +++ b/nes/nc_projections/rotated_nes.py @@ -36,7 +36,7 @@ class RotatedNes(Nes): Tuple with the name of the dimensions of the Longitude values. ('rlat', 'rlon') for a rotated projection. """ - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -52,8 +52,6 @@ class RotatedNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -76,7 +74,7 @@ class RotatedNes(Nes): super(RotatedNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, balanced=balanced, - xarray=xarray, parallel_method=parallel_method, + parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, times=times, **kwargs) @@ -102,7 +100,7 @@ class RotatedNes(Nes): self._lon_dim = ('rlat', 'rlon') @staticmethod - def new(comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def new(comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -118,8 +116,6 @@ class RotatedNes(Nes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -136,7 +132,7 @@ class RotatedNes(Nes): List of times to substitute the current ones while creation. """ - new = RotatedNes(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, + new = RotatedNes(comm=comm, path=path, info=info, dataset=dataset, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, balanced=balanced, times=times, **kwargs) diff --git a/nes/nc_projections/rotated_nested_nes.py b/nes/nc_projections/rotated_nested_nes.py index e56f427..cafc607 100644 --- a/nes/nc_projections/rotated_nested_nes.py +++ b/nes/nc_projections/rotated_nested_nes.py @@ -4,9 +4,10 @@ import numpy as np from netCDF4 import Dataset from .rotated_nes import RotatedNes + class RotatedNestedNes(RotatedNes): - def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + def __init__(self, comm=None, path=None, info=False, dataset=None, parallel_method='Y', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, balanced=False, times=None, **kwargs): """ @@ -22,8 +23,6 @@ class RotatedNestedNes(RotatedNes): Indicates if you want to get reading/writing info. dataset: Dataset NetCDF4-python Dataset to initialize the class. - xarray: bool: - (Not working) Indicates if you want to use xarray as default. parallel_method : str Indicates the parallelization method that you want. Default: 'Y'. Accepted values: ['X', 'Y', 'T']. @@ -46,7 +45,7 @@ class RotatedNestedNes(RotatedNes): super(RotatedNestedNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, balanced=balanced, - xarray=xarray, parallel_method=parallel_method, + parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, first_level=first_level, last_level=last_level, create_nes=create_nes, times=times, **kwargs) diff --git a/requirements.txt b/requirements.txt index a40856e..02666b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,5 +12,5 @@ scipy>=1.7.3 filelock>=3.9.0 eccodes-python~=0.9.5 cfunits>=3.3.5 -xarray>=0.20.2 -mpi4py>=3.1.4 \ No newline at end of file +mpi4py>=3.1.4 +rasterio>=1.1.3 -- GitLab From 58d649a6d5cfe9d29f6ad2a303a7cf70bbe9e7cd Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 3 Jan 2024 15:43:47 +0100 Subject: [PATCH 2/3] self.netcdf to self.dataset --- nes/methods/horizontal_interpolation.py | 2 +- nes/methods/vertical_interpolation.py | 2 +- nes/nc_projections/default_nes.py | 62 ++++++++++---------- nes/nc_projections/points_nes.py | 2 +- nes/nc_projections/points_nes_ghost.py | 2 +- nes/nc_projections/points_nes_providentia.py | 2 +- nes/nes_formats/cmaq_format.py | 2 +- nes/nes_formats/monarch_format.py | 2 +- nes/nes_formats/wrf_chem_format.py | 2 +- 9 files changed, 38 insertions(+), 40 deletions(-) diff --git a/nes/methods/horizontal_interpolation.py b/nes/methods/horizontal_interpolation.py index 82d3141..3690843 100644 --- a/nes/methods/horizontal_interpolation.py +++ b/nes/methods/horizontal_interpolation.py @@ -83,7 +83,7 @@ def interpolate_horizontal(self, dst_grid, weight_matrix_path=None, kind='Neares # Remove original file information final_dst.__ini_path = None - final_dst.netcdf = None + final_dst.dataset = None final_dst.dataset = None # Return final_dst diff --git a/nes/methods/vertical_interpolation.py b/nes/methods/vertical_interpolation.py index 995207b..94ffd51 100644 --- a/nes/methods/vertical_interpolation.py +++ b/nes/methods/vertical_interpolation.py @@ -194,6 +194,6 @@ def interpolate_vertical(self, new_levels, new_src_vertical=None, kind='linear', # Remove original file information self.__ini_path = None - self.netcdf = None + self.dataset = None return self diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 12efaff..151f319 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -40,8 +40,6 @@ class Nes(object): Number of hours to avoid from the first original values. hours_end : int Number of hours to avoid from the last original values. - dataset : xr.Dataset - (not working) xArray Dataset. netcdf : Dataset netcdf4-python Dataset. variables : dict @@ -164,7 +162,7 @@ class Nes(object): # NetCDF object if create_nes: - self.netcdf = None + self.dataset = None # Set string length self.strlen = None @@ -202,7 +200,7 @@ class Nes(object): else: if dataset is not None: - self.netcdf = dataset + self.dataset = dataset elif self.__ini_path is not None: self.open() @@ -320,8 +318,8 @@ class Nes(object): Max length of the string data """ - if 'strlen' in self.netcdf.dimensions: - strlen = self.netcdf.dimensions['strlen'].size + if 'strlen' in self.dataset.dimensions: + strlen = self.dataset.dimensions['strlen'].size else: return None @@ -455,7 +453,7 @@ class Nes(object): """ nessy = deepcopy(self) - nessy.netcdf = None + nessy.dataset = None if copy_vars: nessy.set_communicator(self.comm) nessy.variables = deepcopy(self.variables) @@ -1433,7 +1431,7 @@ class Nes(object): Open the NetCDF. """ - self.netcdf = self.__open_netcdf4() + self.dataset = self.__open_netcdf4() return None @@ -1457,7 +1455,7 @@ class Nes(object): else: netcdf = Dataset(self.__ini_path, format="NETCDF4", mode=mode, parallel=True, comm=self.comm, info=MPI.Info()) - self.netcdf = netcdf + self.dataset = netcdf return netcdf @@ -1469,9 +1467,9 @@ class Nes(object): if self.master: self.serial_nc.close() self.serial_nc = None - if (hasattr(self, 'netcdf')) and (self.netcdf is not None): - self.netcdf.close() - self.netcdf = None + if (hasattr(self, 'netcdf')) and (self.dataset is not None): + self.dataset.close() + self.dataset = None return None @@ -1610,7 +1608,7 @@ class Nes(object): """ if self.master: - nc_var = self.netcdf.variables['time'] + nc_var = self.dataset.variables['time'] time_data, units, calendar = self.__parse_time(nc_var) # Extracting time resolution depending on the units self._time_resolution = self.__get_time_resolution_from_units(units) @@ -1644,12 +1642,12 @@ class Nes(object): if self.master: if not create_nes: - if 'time_bnds' in self.netcdf.variables.keys() or self._climatology: - time = self.netcdf.variables['time'] + if 'time_bnds' in self.dataset.variables.keys() or self._climatology: + time = self.dataset.variables['time'] if self._climatology: - nc_var = self.netcdf.variables[self._climatology_var_name] + nc_var = self.dataset.variables[self._climatology_var_name] else: - nc_var = self.netcdf.variables['time_bnds'] + nc_var = self.dataset.variables['time_bnds'] time_bnds = num2date(nc_var[:], self.__parse_time_unit(time.units), calendar=time.calendar).tolist() else: @@ -1684,12 +1682,12 @@ class Nes(object): if self.master: if not create_nes: - if 'lat_bnds' in self.netcdf.variables.keys(): - lat_bnds = {'data': self._unmask_array(self.netcdf.variables['lat_bnds'][:])} + if 'lat_bnds' in self.dataset.variables.keys(): + lat_bnds = {'data': self._unmask_array(self.dataset.variables['lat_bnds'][:])} else: lat_bnds = None - if 'lon_bnds' in self.netcdf.variables.keys(): - lon_bnds = {'data': self._unmask_array(self.netcdf.variables['lon_bnds'][:])} + if 'lon_bnds' in self.dataset.variables.keys(): + lon_bnds = {'data': self._unmask_array(self.dataset.variables['lon_bnds'][:])} else: lon_bnds = None else: @@ -1723,9 +1721,9 @@ class Nes(object): c_measures = {} if self.master: if not create_nes: - if 'cell_area' in self.netcdf.variables.keys(): + if 'cell_area' in self.dataset.variables.keys(): c_measures['cell_area'] = {} - c_measures['cell_area']['data'] = self._unmask_array(self.netcdf.variables['cell_area'][:]) + c_measures['cell_area']['data'] = self._unmask_array(self.dataset.variables['cell_area'][:]) c_measures = self.comm.bcast(c_measures, root=0) self.free_vars(['cell_area']) @@ -1756,7 +1754,7 @@ class Nes(object): dimension_name = set(possible_names).intersection(set(self.variables.keys())).pop() nc_var = self.variables[dimension_name].copy() - nc_var['data'] = self.netcdf.variables[dimension_name][:] + nc_var['data'] = self.dataset.variables[dimension_name][:] if hasattr(nc_var, 'units'): if nc_var['units'] in ['unitless', '-']: nc_var['units'] = '' @@ -1881,7 +1879,7 @@ class Nes(object): if self.master: variables = {} # Initialise data - for var_name, var_info in self.netcdf.variables.items(): + for var_name, var_info in self.dataset.variables.items(): variables[var_name] = {} variables[var_name]['data'] = None variables[var_name]['dimensions'] = var_info.dimensions @@ -1919,7 +1917,7 @@ class Nes(object): Portion of the variable data corresponding to the rank. """ - nc_var = self.netcdf.variables[var_name] + nc_var = self.dataset.variables[var_name] var_dims = nc_var.dimensions # Read data in 4 dimensions @@ -1994,10 +1992,10 @@ class Nes(object): List (or single string) of the variables to be loaded. """ - if (self.__ini_path is None) and (self.netcdf is None): + if (self.__ini_path is None) and (self.dataset is None): raise RuntimeError('Only data from existing files can be loaded.') - if self.netcdf is None: + if self.dataset is None: self.__open_dataset() close = True else: @@ -2130,8 +2128,8 @@ class Nes(object): gl_attrs = {} if not create_nes: - for attrname in self.netcdf.ncattrs(): - gl_attrs[attrname] = getattr(self.netcdf, attrname) + for attrname in self.dataset.ncattrs(): + gl_attrs[attrname] = getattr(self.dataset, attrname) return gl_attrs @@ -2705,7 +2703,7 @@ class Nes(object): if att_value is not None: if self.info: print("Rank {0:03d}: Filling {1})".format(self.rank, var_name)) - var = self.netcdf.variables[var_name] + var = self.dataset.variables[var_name] if isinstance(att_value, int) and att_value == 0: var[i_time, self.write_axis_limits['z_min']:self.write_axis_limits['z_max'], @@ -2816,7 +2814,7 @@ class Nes(object): netcdf.setncattr('Conventions', 'CF-1.7') if keep_open: - self.netcdf = netcdf + self.dataset = netcdf else: netcdf.close() diff --git a/nes/nc_projections/points_nes.py b/nes/nc_projections/points_nes.py index bca62c0..9df46e2 100644 --- a/nes/nc_projections/points_nes.py +++ b/nes/nc_projections/points_nes.py @@ -310,7 +310,7 @@ class PointsNes(Nes): Portion of the variable data corresponding to the rank. """ - nc_var = self.netcdf.variables[var_name] + nc_var = self.dataset.variables[var_name] var_dims = nc_var.dimensions # Read data in 1 or 2 dimensions diff --git a/nes/nc_projections/points_nes_ghost.py b/nes/nc_projections/points_nes_ghost.py index 2acdd81..b528edf 100644 --- a/nes/nc_projections/points_nes_ghost.py +++ b/nes/nc_projections/points_nes_ghost.py @@ -279,7 +279,7 @@ class PointsNesGHOST(PointsNes): Portion of the variable data corresponding to the rank. """ - nc_var = self.netcdf.variables[var_name] + nc_var = self.dataset.variables[var_name] var_dims = nc_var.dimensions # Read data in 1 or 2 dimensions diff --git a/nes/nc_projections/points_nes_providentia.py b/nes/nc_projections/points_nes_providentia.py index b157cb3..4a72bc0 100644 --- a/nes/nc_projections/points_nes_providentia.py +++ b/nes/nc_projections/points_nes_providentia.py @@ -316,7 +316,7 @@ class PointsNesProvidentia(PointsNes): Portion of the variable data corresponding to the rank. """ - nc_var = self.netcdf.variables[var_name] + nc_var = self.dataset.variables[var_name] var_dims = nc_var.dimensions # Read data in 1, 2 or 3 dimensions diff --git a/nes/nes_formats/cmaq_format.py b/nes/nes_formats/cmaq_format.py index f27bc77..b8b344d 100644 --- a/nes/nes_formats/cmaq_format.py +++ b/nes/nes_formats/cmaq_format.py @@ -59,7 +59,7 @@ def to_netcdf_cmaq(self, path, chunking=False, keep_open=False): # Close NetCDF if keep_open: - self.netcdf = netcdf + self.dataset = netcdf else: netcdf.close() diff --git a/nes/nes_formats/monarch_format.py b/nes/nes_formats/monarch_format.py index 0087472..e660638 100644 --- a/nes/nes_formats/monarch_format.py +++ b/nes/nes_formats/monarch_format.py @@ -73,7 +73,7 @@ def to_netcdf_monarch(self, path, chunking=False, keep_open=False): netcdf.setncattr('Conventions', 'CF-1.7') if keep_open: - self.netcdf = netcdf + self.dataset = netcdf else: netcdf.close() diff --git a/nes/nes_formats/wrf_chem_format.py b/nes/nes_formats/wrf_chem_format.py index 77bf937..1dffc8d 100644 --- a/nes/nes_formats/wrf_chem_format.py +++ b/nes/nes_formats/wrf_chem_format.py @@ -67,7 +67,7 @@ def to_netcdf_wrf_chem(self, path, chunking=False, keep_open=False): # Close NetCDF if keep_open: - self.netcdf = netcdf + self.dataset = netcdf else: netcdf.close() -- GitLab From 3fbbed395fea8f8749529b9bcdc6ce734b4f972f Mon Sep 17 00:00:00 2001 From: ctena Date: Wed, 20 Mar 2024 12:39:00 +0100 Subject: [PATCH 3/3] Trying to undo the mess that I did with the branches --- nes/__init__.py | 2 +- nes/nc_projections/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nes/__init__.py b/nes/__init__.py index 86a702f..51ac4bc 100644 --- a/nes/__init__.py +++ b/nes/__init__.py @@ -2,7 +2,7 @@ __date__ = "2023-06-22" __version__ = "1.1.3" from .load_nes import open_netcdf, concatenate_netcdfs -from .load_nes import open_raster +# from .load_nes import open_raster from .create_nes import create_nes, from_shapefile from .nc_projections import * from .methods.cell_measures import calculate_geometry_area diff --git a/nes/nc_projections/__init__.py b/nes/nc_projections/__init__.py index f5c287b..fa530f9 100644 --- a/nes/nc_projections/__init__.py +++ b/nes/nc_projections/__init__.py @@ -7,4 +7,4 @@ from .points_nes_ghost import PointsNesGHOST from .points_nes_providentia import PointsNesProvidentia from .lcc_nes import LCCNes from .mercator_nes import MercatorNes -from .raster_nes import RasterNes +# from .raster_nes import RasterNes -- GitLab