diff --git a/Jupyter_notebooks/NES_create_netcdf_test.ipynb b/Jupyter_notebooks/NES_create_netcdf_test.ipynb index 188e4b21e641494213a19e4e2d9f98428b32d8b4..e19764cf90651dd90c8abe3cad7e715828646c5e 100644 --- a/Jupyter_notebooks/NES_create_netcdf_test.ipynb +++ b/Jupyter_notebooks/NES_create_netcdf_test.ipynb @@ -7,7 +7,15 @@ "outputs": [], "source": [ "from nes import *\n", - "import xarray as xr" + "import xarray as xr\n", + "from netCDF4 import Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create regular grid" ] }, { @@ -415,7 +423,7 @@ "Data variables:\n", " crs |S1 b''\n", "Attributes:\n", - " Conventions: CF-1.7" + " Conventions: CF-1.7" ], "text/plain": [ "\n", @@ -440,6 +448,13 @@ "xr.open_dataset('regular_grid.nc')" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create rotated grid" + ] + }, { "cell_type": "code", "execution_count": 5, @@ -854,7 +869,7 @@ " lon (rlat, rlon) float64 -35.0 -34.8 -34.6 ... -33.6 -33.4 -33.2\n", " rotated_pole |S1 b''\n", "Attributes:\n", - " Conventions: CF-1.7
  • Conventions :
    CF-1.7
    • " ], "text/plain": [ "\n", @@ -898,6 +913,1299 @@ "source": [ "xr.open_dataset('rotated_grid.nc')" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create grid from random points" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "lat = [0, 10, 23, 50]\n", + "lon = [25, -30, 41, 12]\n", + "points_grid = create_nes(comm=None, info=False, projection=None, create_nes=True,\n", + " lat=lat, lon=lon)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank 000: Creating points_grid.nc\n", + "Rank 000: NetCDF ready to write\n", + "Rank 000: Dimensions done\n" + ] + } + ], + "source": [ + "points_grid.to_netcdf('points_grid.nc', info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset>\n",
+       "Dimensions:  (time: 1, lev: 1, lat: 4, lon: 4)\n",
+       "Coordinates:\n",
+       "  * time     (time) datetime64[ns] 1996-12-31\n",
+       "  * lev      (lev) float64 0.0\n",
+       "  * lat      (lat) float64 0.0 10.0 23.0 50.0\n",
+       "  * lon      (lon) float64 25.0 -30.0 41.0 12.0\n",
+       "Data variables:\n",
+       "    *empty*\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.7
    " + ], + "text/plain": [ + "\n", + "Dimensions: (time: 1, lev: 1, lat: 4, lon: 4)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 1996-12-31\n", + " * lev (lev) float64 0.0\n", + " * lat (lat) float64 0.0 10.0 23.0 50.0\n", + " * lon (lon) float64 25.0 -30.0 41.0 12.0\n", + "Data variables:\n", + " *empty*\n", + "Attributes:\n", + " Conventions: CF-1.7" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xr.open_dataset('points_grid.nc')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create grid from NetCDF file" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset>\n",
+       "Dimensions:                       (station: 8643, time: 744)\n",
+       "Coordinates:\n",
+       "  * time                          (time) datetime64[ns] 2021-07-01 ... 2021-0...\n",
+       "Dimensions without coordinates: station\n",
+       "Data variables: (12/20)\n",
+       "    station_code                  (station) |S75 b'AD0942A' ... b'ES2074A'\n",
+       "    station_start_date            (station) |S75 b'2004-06-17' ... b'nan'\n",
+       "    station_zone                  (station) |S75 b'nan' b'nan' ... b'nature'\n",
+       "    lat                           (station) float32 42.51 42.52 ... 28.09 28.48\n",
+       "    street_type                   (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n",
+       "    country_code                  (station) |S75 b'AD' b'AD' ... b'ES' b'ES'\n",
+       "    ...                            ...\n",
+       "    lon                           (station) float32 1.539 1.565 ... -16.26\n",
+       "    station_end_date              (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n",
+       "    station_local_code            (station) |S75 b'942' b'944' ... b'38038033'\n",
+       "    station_rural_back            (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n",
+       "    station_ozone_classification  (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n",
+       "    pm10                          (time, station) float32 ...
    " + ], + "text/plain": [ + "\n", + "Dimensions: (station: 8643, time: 744)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2021-07-01 ... 2021-0...\n", + "Dimensions without coordinates: station\n", + "Data variables: (12/20)\n", + " station_code (station) |S75 ...\n", + " station_start_date (station) |S75 ...\n", + " station_zone (station) |S75 ...\n", + " lat (station) float32 ...\n", + " street_type (station) |S75 ...\n", + " country_code (station) |S75 ...\n", + " ... ...\n", + " lon (station) float32 ...\n", + " station_end_date (station) |S75 ...\n", + " station_local_code (station) |S75 ...\n", + " station_rural_back (station) |S75 ...\n", + " station_ozone_classification (station) |S75 ...\n", + " pm10 (time, station) float32 ..." + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nc_path = '/esarchive/obs/eea/eionet/hourly/pm10/pm10_202107.nc'\n", + "xr.open_dataset(nc_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "nc = Dataset(nc_path, mode='r', clobber=False)\n", + "lat = nc.variables['lat'][:]\n", + "lon = nc.variables['lon'][:]\n", + "nc_points_grid = create_nes(comm=None, info=False, projection=None, create_nes=True,\n", + " lat=lat, lon=lon)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank 000: Creating nc_points_grid.nc\n", + "Rank 000: NetCDF ready to write\n", + "Rank 000: Dimensions done\n" + ] + } + ], + "source": [ + "nc_points_grid.to_netcdf('nc_points_grid.nc', info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset>\n",
+       "Dimensions:  (time: 1, lev: 1, lat: 8643, lon: 8643)\n",
+       "Coordinates:\n",
+       "  * time     (time) datetime64[ns] 1996-12-31\n",
+       "  * lev      (lev) float64 0.0\n",
+       "  * lat      (lat) float64 42.51 42.52 42.53 41.33 ... 36.96 40.95 28.09 28.48\n",
+       "  * lon      (lon) float64 1.539 1.565 1.717 19.82 ... -0.2908 -17.12 -16.26\n",
+       "Data variables:\n",
+       "    *empty*\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.7
    " + ], + "text/plain": [ + "\n", + "Dimensions: (time: 1, lev: 1, lat: 8643, lon: 8643)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 1996-12-31\n", + " * lev (lev) float64 0.0\n", + " * lat (lat) float64 42.51 42.52 42.53 41.33 ... 36.96 40.95 28.09 28.48\n", + " * lon (lon) float64 1.539 1.565 1.717 19.82 ... -0.2908 -17.12 -16.26\n", + "Data variables:\n", + " *empty*\n", + "Attributes:\n", + " Conventions: CF-1.7" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xr.open_dataset('nc_points_grid.nc')" + ] } ], "metadata": { diff --git a/Jupyter_notebooks/NES_time_bnds_test.ipynb b/Jupyter_notebooks/NES_time_bnds_test.ipynb index caf5608bb3558abe74c990d0f7676221b8bcdb0a..1751bf61e7008473953defbfabd828e224f2e8b8 100644 --- a/Jupyter_notebooks/NES_time_bnds_test.ipynb +++ b/Jupyter_notebooks/NES_time_bnds_test.ipynb @@ -13,29 +13,10 @@ ] }, { - "cell_type": "code", - "execution_count": 2, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'\\ntest = \"/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc\"\\nnessy = open_netcdf(path=test, info=True)\\nnessy.keep_vars([\\'O3_all\\'])\\nnessy.load()\\nnessy.to_netcdf(\\'nc_serial_test2_alba.nc\\', info=True)\\n'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "\"\"\"\n", - "test = \"/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc\"\n", - "nessy = open_netcdf(path=test, info=True)\n", - "nessy.keep_vars(['O3_all'])\n", - "nessy.load()\n", - "nessy.to_netcdf('nc_serial_test2_alba.nc', info=True)\n", - "\"\"\"" + "# Set time bounds" ] }, { @@ -44,8 +25,8 @@ "metadata": {}, "outputs": [], "source": [ - "test = \"/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc\"\n", - "nessy = open_netcdf(path=test, info=True)" + "test_path = \"/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc\"\n", + "nessy = open_netcdf(path=test_path, info=True)" ] }, { @@ -72,25 +53,6 @@ "nessy.time_bnds" ] }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/default_nes.py:1136: UserWarning: WARNING!!! Variable O3_all was not loaded. It will not be written.\n", - " warnings.warn(msg)\n" - ] - } - ], - "source": [ - "# nessy.load()\n", - "nessy.to_netcdf('nc_serial_test3_alba.nc')" - ] - }, { "cell_type": "code", "execution_count": 6, @@ -115,7 +77,14 @@ "metadata": {}, "outputs": [], "source": [ - "nessy.to_netcdf('nc_serial_test3_alba.nc')" + "nessy.to_netcdf('nc_serial_test.nc')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Explore variables" ] }, { @@ -678,15 +647,8 @@ } ], "source": [ - "xr.open_dataset('nc_serial_test3_alba.nc')" + "xr.open_dataset('nc_serial_test.nc')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/nes/create_nes.py b/nes/create_nes.py index 9bea9ea8ba8c5632680cbc2a40e3216f903dc319..9aa278b8af9d37c87aa59802b1b6fb20e9454783 100644 --- a/nes/create_nes.py +++ b/nes/create_nes.py @@ -7,6 +7,7 @@ from mpi4py import MPI import warnings from nes.nc_projections.rotated_nes import RotatedNes from nes.nc_projections.latlon_nes import LatLonNes +from nes.nc_projections.points_nes import PointsNes def create_nes(comm=None, info=False, projection=None, create_nes=True, **kwargs): @@ -20,7 +21,9 @@ def create_nes(comm=None, info=False, projection=None, create_nes=True, **kwargs for name, value in kwargs.items(): kwargs_list.append(name) - if projection == 'regular': + if projection==None: + required_vars = ['lat', 'lon'] + elif projection == 'regular': required_vars = ['lat_orig', 'lon_orig', 'inc_lat', 'inc_lon', 'n_lat', 'n_lon'] elif projection == 'rotated': required_vars = ['centre_lat', 'centre_lon', 'west_boundary', 'south_boundary', 'inc_rlat', 'inc_rlon', @@ -36,7 +39,12 @@ def create_nes(comm=None, info=False, projection=None, create_nes=True, **kwargs msg += 'Variable {0} has not been defined.'.format(var) warnings.warn(msg) - if projection == 'regular': + if projection == None: + nessy = PointsNes(comm=comm, dataset=None, xarray=None, info=info, parallel_method=None, + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, + create_nes=create_nes, **kwargs) + + elif projection == 'regular': nessy = LatLonNes(comm=comm, dataset=None, xarray=None, info=info, parallel_method=None, avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=create_nes, **kwargs) diff --git a/nes/nc_projections/latlon_nes.py b/nes/nc_projections/latlon_nes.py index 68e49b9fb3a0fcf25ca6a9279ba120dc9303783a..1fda96ea3615dbcebe614859e7dd779e3b9b5fed 100644 --- a/nes/nc_projections/latlon_nes.py +++ b/nes/nc_projections/latlon_nes.py @@ -36,13 +36,13 @@ class LatLonNes(Nes): NetCDF4-python Dataset to initialize the class xarray: bool: (Not working) Indicates if you want to use xarray as default + parallel_method : str + Indicates the parallelization method that you want. Default over Y axis + accepted values: ['Y', 'T'] avoid_first_hours : int Number of hours to remove from first time steps. avoid_last_hours : int Number of hours to remove from last time steps. - parallel_method : str - Indicates the parallelization method that you want. Default over Y axis - accepted values: ['Y', 'T'] """ super(LatLonNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, parallel_method=parallel_method, @@ -76,7 +76,14 @@ class LatLonNes(Nes): return None def _create_centroids(self, **kwargs): - + """ + Calculate center latitudes and longitudes from grid details. + + Parameters + ---------- + netcdf : Dataset + NetCDF object. + """ # Calculate center latitudes lat_c_orig = kwargs['lat_orig'] + (kwargs['inc_lat'] / 2) self.center_lats = np.linspace( diff --git a/nes/nc_projections/points_nes.py b/nes/nc_projections/points_nes.py new file mode 100644 index 0000000000000000000000000000000000000000..316d0db67a03661982518134a3b9559c5e172e74 --- /dev/null +++ b/nes/nc_projections/points_nes.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +from nes.nc_projections.default_nes import Nes + +class PointsNes(Nes): + """ + + Attributes + ---------- + _var_dim : tuple + Tuple with the name of the Y and X dimensions for the variables. + ('lat', 'lon') for a regular latitude-longitude projection. + _lat_dim : tuple + Tuple with the name of the dimensions of the Latitude values. + ('lat',) for a regular latitude-longitude projection. + _lon_dim : tuple + Tuple with the name of the dimensions of the Longitude values. + ('lon',) for a regular latitude-longitude projection. + """ + def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, + **kwargs): + """ + Initialize the PointsNes class + + Parameters + ---------- + comm: MPI.COMM + Path to the CSV file that contains all the information. + path: str + Path to the NetCDF to initialize the object + info: bool + Indicates if you want to get reading/writing info + dataset: Dataset + NetCDF4-python Dataset to initialize the class + xarray: bool: + (Not working) Indicates if you want to use xarray as default + parallel_method : str + Indicates the parallelization method that you want. Default over Y axis + accepted values: ['Y', 'T'] + avoid_first_hours : int + Number of hours to remove from first time steps. + avoid_last_hours : int + Number of hours to remove from last time steps. + """ + super(PointsNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, + xarray=xarray, parallel_method=parallel_method, + avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, + first_level=first_level, last_level=last_level, create_nes=create_nes, + **kwargs) + + if create_nes: + self._lat, self._lon = self._create_centroids(**kwargs) + + self._var_dim = ('lat', 'lon') + self._lat_dim = ('lat',) + self._lon_dim = ('lon',) + + def _create_dimensions(self, netcdf): + """ + Create the 'lat', 'lon' dimensions and the super dimensions ('lev', 'time'). + + Parameters + ---------- + netcdf : Dataset + NetCDF object. + """ + super(PointsNes, self)._create_dimensions(netcdf) + + netcdf.createDimension('lon', len(self._lon['data'])) + netcdf.createDimension('lat', len(self._lat['data'])) + + return None + + def _create_centroids(self, **kwargs): + """ + Calculate center latitudes and longitudes from points. + + Parameters + ---------- + netcdf : Dataset + NetCDF object. + """ + # Calculate center latitudes + self.center_lats = kwargs['lat'] + + # Calculate center longitudes + self.center_lons = kwargs['lon'] + + return {'data': self.center_lats}, {'data': self.center_lons} \ No newline at end of file diff --git a/nes/nc_projections/rotated_nes.py b/nes/nc_projections/rotated_nes.py index 1a8268e4cbe0c1d88acca35e32f9e875f2879710..c027ab2c7e5d23dd6fb5871a32de2da579647818 100644 --- a/nes/nc_projections/rotated_nes.py +++ b/nes/nc_projections/rotated_nes.py @@ -48,13 +48,13 @@ class RotatedNes(Nes): NetCDF4-python Dataset to initialize the class xarray: bool: (Not working) Indicates if you want to use xarray as default + parallel_method : str + Indicates the parallelization method that you want. Default over Y axis + accepted values: ['Y', 'T'] avoid_first_hours : int Number of hours to remove from first time steps. avoid_last_hours : int Number of hours to remove from last time steps. - parallel_method : str - Indicates the parallelization method that you want. Default over Y axis - accepted values: ['Y', 'T'] """ super(RotatedNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, @@ -150,7 +150,14 @@ class RotatedNes(Nes): return None def _create_centroids(self, **kwargs): - + """ + Calculate center latitudes and longitudes from grid details. + + Parameters + ---------- + netcdf : Dataset + NetCDF object. + """ # Calculate center latitudes self.center_lats = np.linspace(kwargs['south_boundary'], kwargs['south_boundary'] + (kwargs['inc_rlat'] * (kwargs['n_lat'] - 1)), kwargs['n_lat'])