diff --git a/jupyter_notebooks/jupyter_bash_nord3v2.cmd b/jupyter_notebooks/jupyter_bash_nord3v2.cmd new file mode 100644 index 0000000000000000000000000000000000000000..805ec0d96a6743021807a229247161077d8d9054 --- /dev/null +++ b/jupyter_notebooks/jupyter_bash_nord3v2.cmd @@ -0,0 +1,36 @@ +#!/bin/bash +#SBATCH --ntasks 1 +#SBATCH --time 03:00:00 +#SBATCH --job-name jupyter-notebook +#SBATCH --output logs/log_jupyter-notebook-%J.out +#SBATCH --error logs/log_jupyter-notebook-%J.err +#SBATCH --exclusive + +# get tunneling info +XDG_RUNTIME_DIR="" +port=$(shuf -i8000-9999 -n1) +node=$(hostname -s) +user=$(whoami) + +# print tunneling instructions jupyter-log +echo -e " + +MacOS or linux terminal command to create your ssh tunnel +ssh -N -L ${port}:${node}:${port} ${user}@nord4.bsc.es + +Use a Browser on your local machine to go to: +localhost:${port} (prefix w/ https:// if using password) +" + +# load modules or conda environments here +module load jupyterlab/3.0.9-foss-2019b-Python-3.7.4 +module load Python/3.7.4-GCCcore-8.3.0 +module load netcdf4-python/1.5.3-foss-2019b-Python-3.7.4 +module load cfunits/1.8-foss-2019b-Python-3.7.4 +module load xarray/0.19.0-foss-2019b-Python-3.7.4 + +export PYTHONPATH=/esarchive/scratch/avilanova/software/NES:${PYTHONPATH} + +# DON'T USE ADDRESS BELOW. +# DO USE TOKEN BELOW +jupyter-lab --no-browser --port=${port} --ip=${node} \ No newline at end of file diff --git a/jupyter_notebooks/jupyter_test_create_netcdf.ipynb b/jupyter_notebooks/jupyter_test_create_netcdf.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..911232c580ab13c20115d152f7b7752e753a765a --- /dev/null +++ b/jupyter_notebooks/jupyter_test_create_netcdf.ipynb @@ -0,0 +1,129 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from nes import *\n", + "import xarray as xr" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "'NoneType' object does not support item deletion", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m regular_grid = create_nes(comm=None, info=False, projection='regular', create_nes=True,\n\u001b[1;32m 8\u001b[0m \u001b[0mlat_orig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlat_orig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlon_orig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlon_orig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minc_lat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minc_lat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minc_lon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minc_lon\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m n_lat=n_lat, n_lon=n_lon)\n\u001b[0m", + "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/create_nes.py\u001b[0m in \u001b[0;36mcreate_nes\u001b[0;34m(comm, info, projection, create_nes, **kwargs)\u001b[0m\n\u001b[1;32m 40\u001b[0m nessy = LatLonNes(comm=comm, dataset=None, xarray=None, info=info, parallel_method=None,\n\u001b[1;32m 41\u001b[0m \u001b[0mavoid_first_hours\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mavoid_last_hours\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfirst_level\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_level\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m create_nes=create_nes, **kwargs)\n\u001b[0m\u001b[1;32m 43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mprojection\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'rotated'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/latlon_nes.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, comm, path, info, dataset, xarray, parallel_method, avoid_first_hours, avoid_last_hours, first_level, last_level, create_nes, **kwargs)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0mavoid_first_hours\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mavoid_first_hours\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mavoid_last_hours\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mavoid_last_hours\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0mfirst_level\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfirst_level\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_level\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlast_level\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_nes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_nes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m **kwargs)\n\u001b[0m\u001b[1;32m 52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcreate_nes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/default_nes.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, comm, path, info, dataset, xarray, parallel_method, avoid_first_hours, avoid_last_hours, first_level, last_level, create_nes, **kwargs)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcreate_nes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 133\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_time_bnds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_time_bnds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcreate_nes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/default_nes.py\u001b[0m in \u001b[0;36m__get_time\u001b[0;34m(self, create_nes)\u001b[0m\n\u001b[1;32m 692\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 693\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mroot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 694\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfree_vars\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 695\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 696\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/default_nes.py\u001b[0m in \u001b[0;36mfree_vars\u001b[0;34m(self, var_list)\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mvar_name\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvar_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvar_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 332\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: 'NoneType' object does not support item deletion" + ] + } + ], + "source": [ + "lat_orig = 41.1\n", + "lon_orig = 1.8\n", + "inc_lat = 0.1\n", + "inc_lon = 0.1\n", + "n_lat = 10\n", + "n_lon = 10\n", + "regular_grid = create_nes(comm=None, info=False, projection='regular', create_nes=True,\n", + " lat_orig=lat_orig, lon_orig=lon_orig, inc_lat=inc_lat, inc_lon=inc_lon, \n", + " n_lat=n_lat, n_lon=n_lon)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "regular_grid.to_netcdf('regular_grid.nc', info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xr.open_dataset('regular_grid.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "centre_lat = 51\n", + "centre_lon = 10\n", + "west_boundary = -35\n", + "south_boundary = -27\n", + "inc_rlat = 0.2\n", + "inc_rlon = 0.2\n", + "n_lat = 10\n", + "n_lon = 10\n", + "grid_north_pole_latitude = 90\n", + "grid_north_pole_longitude = 135\n", + "rotated_grid = create_nes(comm=None, info=False, projection='rotated', create_nes=True,\n", + " centre_lat=centre_lat, centre_lon=centre_lon,\n", + " west_boundary=west_boundary, south_boundary=south_boundary,\n", + " inc_rlat=inc_rlat, inc_rlon=inc_rlon, n_lat=n_lat, n_lon=n_lon, \n", + " grid_north_pole_latitude=grid_north_pole_latitude,\n", + " grid_north_pole_longitude=grid_north_pole_longitude)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rotated_grid.to_netcdf('rotated_grid.nc', info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xr.open_dataset('rotated_grid.nc')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/jupyter_notebooks/jupyter_test_time_bnds.ipynb b/jupyter_notebooks/jupyter_test_time_bnds.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..088acedf1afc08891386f4a5eb774a389947e2ac --- /dev/null +++ b/jupyter_notebooks/jupyter_test_time_bnds.ipynb @@ -0,0 +1,721 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from nes import *\n", + "import xarray as xr\n", + "import datetime\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\ntest = \"/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc\"\\nnessy = open_netcdf(path=test, info=True)\\nnessy.keep_vars([\\'O3_all\\'])\\nnessy.load()\\nnessy.to_netcdf(\\'nc_serial_test2_alba.nc\\', info=True)\\n'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "test = \"/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc\"\n", + "nessy = open_netcdf(path=test, info=True)\n", + "nessy.keep_vars(['O3_all'])\n", + "nessy.load()\n", + "nessy.to_netcdf('nc_serial_test2_alba.nc', info=True)\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': None, 'dimensions': (), 'grid_mapping_name': 'rotated_latitude_longitude', 'grid_north_pole_latitude': 39.0, 'grid_north_pole_longitude': -170.0} \n" + ] + } + ], + "source": [ + "test = \"/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc\"\n", + "nessy = open_netcdf(path=test, info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[datetime.datetime(2020, 2, 20, 0, 0),\n", + " datetime.datetime(2020, 2, 15, 0, 0)]], dtype=object)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "array = np.array([[datetime.datetime(year=2020, month=2, day=20), \n", + " datetime.datetime(year=2020, month=2, day=15)]])\n", + "nessy.set_time_bnds(array)\n", + "nessy.time_bnds" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/default_nes.py:970: UserWarning: WARNING!!! Variable O3_all was not loaded. It will not be written.\n", + " warnings.warn(msg)\n" + ] + } + ], + "source": [ + "# nessy.load()\n", + "nessy.to_netcdf('nc_serial_test3_alba.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank 000: Loading O3_all var (1/1)\n", + "Rank 000: Loaded O3_all var ((1, 24, 271, 351))\n" + ] + } + ], + "source": [ + "nessy.load()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "nessy.to_netcdf('nc_serial_test3_alba.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': masked_array(\n", + " data=[[[[2.82636070e-09, 2.83436141e-09, 2.82522827e-09, ...,\n", + " 2.96334601e-09, 2.94810221e-09, 2.91839553e-09],\n", + " [2.80930834e-09, 2.57182142e-09, 2.55521360e-09, ...,\n", + " 2.56395216e-09, 2.55890820e-09, 2.89073032e-09],\n", + " [2.79031842e-09, 2.53415999e-09, 2.50317100e-09, ...,\n", + " 2.56737764e-09, 2.58685584e-09, 2.85498758e-09],\n", + " ...,\n", + " [4.54469973e-09, 2.31674457e-09, 2.22753971e-09, ...,\n", + " 3.90127353e-09, 3.89643118e-09, 3.95452204e-09],\n", + " [4.54129534e-09, 3.39469808e-09, 2.30205255e-09, ...,\n", + " 3.88824706e-09, 3.88372090e-09, 3.95252631e-09],\n", + " [4.55012028e-09, 4.54941684e-09, 4.55885596e-09, ...,\n", + " 3.93945099e-09, 3.94256938e-09, 3.94736510e-09]],\n", + " \n", + " [[1.61966751e-09, 1.62850033e-09, 1.62801062e-09, ...,\n", + " 1.74583636e-09, 1.74684045e-09, 1.74125825e-09],\n", + " [1.60704539e-09, 1.41438683e-09, 1.39824063e-09, ...,\n", + " 1.43241041e-09, 1.45136980e-09, 1.73744363e-09],\n", + " [1.59303792e-09, 1.41264567e-09, 1.43958856e-09, ...,\n", + " 1.43522705e-09, 1.45869528e-09, 1.72746673e-09],\n", + " ...,\n", + " [3.39471939e-09, 2.65527422e-09, 2.22850582e-09, ...,\n", + " 3.00350167e-09, 3.02176750e-09, 3.04009262e-09],\n", + " [3.42592332e-09, 2.81851942e-09, 2.28753505e-09, ...,\n", + " 2.99818836e-09, 2.99247205e-09, 3.04403525e-09],\n", + " [3.43113582e-09, 3.43824125e-09, 3.44929552e-09, ...,\n", + " 3.05421777e-09, 3.04752024e-09, 3.04445491e-09]],\n", + " \n", + " [[6.52169652e-10, 6.62677024e-10, 6.71934786e-10, ...,\n", + " 6.84429291e-10, 6.85826118e-10, 6.81504464e-10],\n", + " [6.54959087e-10, 6.65219158e-10, 6.72430500e-10, ...,\n", + " 7.02121916e-10, 6.88325397e-10, 6.78990253e-10],\n", + " [6.57915333e-10, 6.72102929e-10, 6.82566170e-10, ...,\n", + " 7.10820458e-10, 7.07094217e-10, 6.77522760e-10],\n", + " ...,\n", + " [2.26027863e-09, 2.27629537e-09, 2.22616392e-09, ...,\n", + " 1.80253423e-09, 1.80225357e-09, 1.75757697e-09],\n", + " [2.25028196e-09, 2.24872521e-09, 2.25445618e-09, ...,\n", + " 1.78916737e-09, 1.75583581e-09, 1.73717007e-09],\n", + " [2.25827335e-09, 2.26974151e-09, 2.28325270e-09, ...,\n", + " 1.80090465e-09, 1.77703174e-09, 1.75434933e-09]],\n", + " \n", + " ...,\n", + " \n", + " [[6.20177729e-11, 6.26959387e-11, 6.28658792e-11, ...,\n", + " 7.74274672e-11, 7.81546980e-11, 7.60479180e-11],\n", + " [6.20486787e-11, 4.91600684e-11, 4.88878833e-11, ...,\n", + " 8.30884250e-11, 8.02152303e-11, 7.64004970e-11],\n", + " [6.20976950e-11, 4.84989236e-11, 4.85273696e-11, ...,\n", + " 8.46209977e-11, 8.60716498e-11, 9.29777644e-11],\n", + " ...,\n", + " [6.15721710e-11, 5.85051035e-11, 5.68927752e-11, ...,\n", + " 7.66955388e-11, 7.87262894e-11, 8.41871295e-11],\n", + " [6.17081941e-11, 5.77536560e-11, 5.71826440e-11, ...,\n", + " 8.49015233e-11, 8.82505458e-11, 9.20043208e-11],\n", + " [6.09760506e-11, 6.03529102e-11, 6.24047411e-11, ...,\n", + " 9.69636524e-11, 9.73700426e-11, 9.67554162e-11]],\n", + " \n", + " [[6.17567178e-11, 6.23894963e-11, 6.25706292e-11, ...,\n", + " 9.04916420e-11, 8.90077803e-11, 8.43536768e-11],\n", + " [6.17901147e-11, 4.59270816e-11, 4.57923699e-11, ...,\n", + " 1.06383589e-10, 1.05693093e-10, 9.44862175e-11],\n", + " [6.18271337e-11, 4.17853495e-11, 3.94594427e-11, ...,\n", + " 1.34135009e-10, 1.37096737e-10, 1.13853482e-10],\n", + " ...,\n", + " [5.87425456e-11, 5.60845814e-11, 5.33429169e-11, ...,\n", + " 6.52061183e-11, 6.64711411e-11, 7.06842501e-11],\n", + " [5.92315016e-11, 5.72428251e-11, 5.51245403e-11, ...,\n", + " 7.10893150e-11, 7.38196310e-11, 7.53354532e-11],\n", + " [5.72967125e-11, 5.87497967e-11, 6.08200851e-11, ...,\n", + " 7.97847274e-11, 8.28124236e-11, 7.89215707e-11]],\n", + " \n", + " [[6.15217946e-11, 6.21571961e-11, 6.23377391e-11, ...,\n", + " 1.08401239e-10, 1.07494236e-10, 1.08711720e-10],\n", + " [6.15563989e-11, 4.56989759e-11, 4.46428450e-11, ...,\n", + " 1.30999808e-10, 1.26581134e-10, 1.39005307e-10],\n", + " [6.15933693e-11, 3.98656906e-11, 3.75483949e-11, ...,\n", + " 1.37105632e-10, 1.48587462e-10, 1.83946344e-10],\n", + " ...,\n", + " [4.68582569e-11, 4.44464673e-11, 4.43960736e-11, ...,\n", + " 5.86025117e-11, 5.84869791e-11, 6.32652056e-11],\n", + " [4.99817097e-11, 4.49490271e-11, 4.43218864e-11, ...,\n", + " 6.19639479e-11, 6.07859180e-11, 6.55651922e-11],\n", + " [4.98553143e-11, 4.61104453e-11, 4.96835975e-11, ...,\n", + " 6.42673414e-11, 6.38328765e-11, 6.38894007e-11]]]],\n", + " mask=False,\n", + " fill_value=1e+20,\n", + " dtype=float32),\n", + " 'dimensions': ('time', 'lev', 'rlat', 'rlon'),\n", + " 'units': 'kg/m3',\n", + " 'long_name': 'TRACERS_044',\n", + " 'coordinates': 'lat lon',\n", + " 'cell_methods': 'time: maximum (interval: 1hr)',\n", + " 'grid_mapping': 'rotated_pole'}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nessy.variables['O3_all']" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[datetime.datetime(2020, 2, 20, 0, 0),\n", + " datetime.datetime(2020, 2, 15, 0, 0)]], dtype=object)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nessy.time_bnds" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:       (time: 1, time_nv: 2, lev: 24, rlat: 271, rlon: 351)\n",
+       "Coordinates:\n",
+       "  * time          (time) datetime64[ns] 2021-08-03\n",
+       "  * lev           (lev) float64 0.0 1.0 2.0 3.0 4.0 ... 19.0 20.0 21.0 22.0 23.0\n",
+       "    lat           (rlat, rlon) float64 16.35 16.43 16.52 ... 58.83 58.68 58.53\n",
+       "    lon           (rlat, rlon) float64 -22.18 -22.02 -21.85 ... 88.05 88.23\n",
+       "  * rlat          (rlat) float64 -27.0 -26.8 -26.6 -26.4 ... 26.4 26.6 26.8 27.0\n",
+       "  * rlon          (rlon) float64 -35.0 -34.8 -34.6 -34.4 ... 34.4 34.6 34.8 35.0\n",
+       "Dimensions without coordinates: time_nv\n",
+       "Data variables:\n",
+       "    time_bnds     (time, time_nv) datetime64[ns] 2020-02-20 2020-02-15\n",
+       "    O3_all        (time, lev, rlat, rlon) float32 ...\n",
+       "    rotated_pole  |S1 b''\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.7\n",
+       "    comment:      Generated on marenostrum4
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 1, time_nv: 2, lev: 24, rlat: 271, rlon: 351)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2021-08-03\n", + " * lev (lev) float64 0.0 1.0 2.0 3.0 4.0 ... 19.0 20.0 21.0 22.0 23.0\n", + " lat (rlat, rlon) float64 ...\n", + " lon (rlat, rlon) float64 ...\n", + " * rlat (rlat) float64 -27.0 -26.8 -26.6 -26.4 ... 26.4 26.6 26.8 27.0\n", + " * rlon (rlon) float64 -35.0 -34.8 -34.6 -34.4 ... 34.4 34.6 34.8 35.0\n", + "Dimensions without coordinates: time_nv\n", + "Data variables:\n", + " time_bnds (time, time_nv) datetime64[ns] ...\n", + " O3_all (time, lev, rlat, rlon) float32 ...\n", + " rotated_pole |S1 ...\n", + "Attributes:\n", + " Conventions: CF-1.7\n", + " comment: Generated on marenostrum4" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xr.open_dataset('nc_serial_test3_alba.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/jupyter_notebooks/nc_serial_test3_alba.nc b/jupyter_notebooks/nc_serial_test3_alba.nc new file mode 100644 index 0000000000000000000000000000000000000000..de4f352c0ae3c2cb6cbbf2f879215ffad5036091 Binary files /dev/null and b/jupyter_notebooks/nc_serial_test3_alba.nc differ diff --git a/jupyter_notebooks/regular_grid.nc b/jupyter_notebooks/regular_grid.nc new file mode 100644 index 0000000000000000000000000000000000000000..40b2f7e8f289b511e1ba344f5439f985c0d43b88 Binary files /dev/null and b/jupyter_notebooks/regular_grid.nc differ diff --git a/jupyter_notebooks/rotated_grid.nc b/jupyter_notebooks/rotated_grid.nc new file mode 100644 index 0000000000000000000000000000000000000000..bfb2a3580c13208f7816904f1a1f3403a2d04546 Binary files /dev/null and b/jupyter_notebooks/rotated_grid.nc differ diff --git a/nes/__init__.py b/nes/__init__.py index c91cab9952be3171cd2733482352f949d4253e94..b0c2446d2fc02b924aa1c25123f66f90838d86b9 100644 --- a/nes/__init__.py +++ b/nes/__init__.py @@ -2,3 +2,4 @@ __date__ = "2022-MM-DD" __version__ = "0.0.1" from nes.load_nes import open_netcdf +from nes.create_nes import create_nes diff --git a/nes/create_nes.py b/nes/create_nes.py new file mode 100644 index 0000000000000000000000000000000000000000..9bea9ea8ba8c5632680cbc2a40e3216f903dc319 --- /dev/null +++ b/nes/create_nes.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +import sys +import os +from venv import create +from mpi4py import MPI +import warnings +from nes.nc_projections.rotated_nes import RotatedNes +from nes.nc_projections.latlon_nes import LatLonNes + +def create_nes(comm=None, info=False, projection=None, create_nes=True, **kwargs): + + if comm is None: + comm = MPI.COMM_WORLD + else: + comm = comm + + # Check if the parameters that are required to create the object have been defined in kwargs + kwargs_list = [] + for name, value in kwargs.items(): + kwargs_list.append(name) + + if projection == 'regular': + required_vars = ['lat_orig', 'lon_orig', 'inc_lat', 'inc_lon', 'n_lat', 'n_lon'] + elif projection == 'rotated': + required_vars = ['centre_lat', 'centre_lon', 'west_boundary', 'south_boundary', 'inc_rlat', 'inc_rlon', + 'n_lat', 'n_lon', 'grid_north_pole_latitude', 'grid_north_pole_longitude'] + elif projection == 'lcc': + required_vars = ['lat_1', 'lat_2', 'lon_0', 'lat_0', 'nx', 'ny', 'inc_x', 'inc_y', 'x_0', 'y_0'] + elif projection == 'mercator': + required_vars = ['lat_ts', 'lon_0', 'nx', 'ny', 'inc_x', 'inc_y', 'x_0', 'y_0'] + + for var in required_vars: + if var not in kwargs_list: + msg = 'WARNING!!! ' + msg += 'Variable {0} has not been defined.'.format(var) + warnings.warn(msg) + + if projection == 'regular': + nessy = LatLonNes(comm=comm, dataset=None, xarray=None, info=info, parallel_method=None, + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, + create_nes=create_nes, **kwargs) + + elif projection == 'rotated': + nessy = RotatedNes(comm=comm, dataset=None, xarray=None, info=info, parallel_method=None, + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, + create_nes=create_nes, **kwargs) + + return nessy \ No newline at end of file diff --git a/nes/load_nes.py b/nes/load_nes.py index c5e824004905a1eb5ec3c34c78499cf7125442c2..699275b9c6ec47ad450fc35afa3f3ceb2cd9ebb8 100644 --- a/nes/load_nes.py +++ b/nes/load_nes.py @@ -2,6 +2,7 @@ import sys import os +from venv import create from mpi4py import MPI from netCDF4 import Dataset from nes.nc_projections.default_nes import Nes @@ -10,7 +11,8 @@ from nes.nc_projections.latlon_nes import LatLonNes def open_netcdf(path, comm=None, xarray=False, info=False, parallel_method='Y', - avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None): + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, + create_nes=False): """ Open a netCDF file @@ -54,16 +56,14 @@ def open_netcdf(path, comm=None, xarray=False, info=False, parallel_method='Y', if __is_rotated(dataset): nessy = RotatedNes(comm=comm, dataset=dataset, xarray=xarray, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, - first_level=first_level, last_level=last_level - ) + first_level=first_level, last_level=last_level, create_nes=create_nes) else: nessy = LatLonNes(comm=comm, dataset=dataset, xarray=xarray, info=info, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, - first_level=first_level, last_level=last_level) + first_level=first_level, last_level=last_level, create_nes=create_nes) return nessy - def __is_rotated(dataset): """ Check if the netCDF is in rotated pole projection or not. @@ -82,5 +82,3 @@ def __is_rotated(dataset): return True else: return False - - diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index b6619710d616ad1c8f14be2ae74823790cca411d..4bee1a0718919d75d6bb0217637dde7590ca4b64 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -79,7 +79,8 @@ class Nes(object): Tuple with the name of the dimensions of the Longitude values """ def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', - avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create=False): + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, + **kwargs): """ Initialize the Nes class @@ -125,47 +126,61 @@ class Nes(object): self.last_level = last_level # NetCDF object - if dataset is not None: - if self.is_xarray: - self.dataset = dataset - self.netcdf = None - else: - self.dataset = None - self.netcdf = dataset - elif self.__ini_path is not None: - if self.is_xarray: - self.dataset = self.__open_dataset() - self.netcdf = None - else: - self.dataset = None - self.netcdf = self.__open_netcdf4() + if create_nes: + self.variables = None - # Lazy variables - self.variables = self.__get_lazy_variables() + self._time = self.__get_time(create_nes) + self._time_bnds = self.__get_time_bnds(create_nes) - # Complete dimension - self._time = self.__get_time() - self._lev = self._get_coordinate_dimension(['lev', 'level', 'lm']) - self._lat = self._get_coordinate_dimension(['lat', 'latitude']) - self._lon = self._get_coordinate_dimension(['lon', 'longitude']) - self._time_bnds = self.__get_time_bnds() + self._lev = {'data': np.array([0]), + 'units': '', + 'positive': 'up'} + self.lev = deepcopy(self._lev) - # Axis limits - self.parallel_method = parallel_method + self._lat, self._lon = self._create_centroids(**kwargs) - self.read_axis_limits = self.set_read_axis_limits() + self.global_attrs = self.__get_global_attributes(create_nes) + else: + if dataset is not None: + if self.is_xarray: + self.dataset = dataset + self.netcdf = None + else: + self.dataset = None + self.netcdf = dataset + elif self.__ini_path is not None: + if self.is_xarray: + self.dataset = self.__open_dataset() + self.netcdf = None + else: + self.dataset = None + self.netcdf = self.__open_netcdf4() - # Dimensions screening - self.time = self._time[self.read_axis_limits['t_min']:self.read_axis_limits['t_max']] - self.lev = self._get_coordinate_values(self._lev, 'Z') - self.lat = self._get_coordinate_values(self._lat, 'Y') - self.lon = self._get_coordinate_values(self._lon, 'X') - self.time_bnds = self._time_bnds + # Lazy variables + self.variables = self.__get_lazy_variables() - self.write_axis_limits = self.set_write_axis_limits() + # Complete dimension + self._time = self.__get_time() + self._lev = self._get_coordinate_dimension(['lev', 'level', 'lm']) + self._lat = self._get_coordinate_dimension(['lat', 'latitude']) + self._lon = self._get_coordinate_dimension(['lon', 'longitude']) + self._time_bnds = self.__get_time_bnds() - # NetCDF attributes - self.global_attrs = self.__get_global_attributes() + # Axis limits + self.parallel_method = parallel_method + self.read_axis_limits = self.set_read_axis_limits() + + # Dimensions screening + self.time = self._time[self.read_axis_limits['t_min']:self.read_axis_limits['t_max']] + self.lev = self._get_coordinate_values(self._lev, 'Z') + self.lat = self._get_coordinate_values(self._lat, 'Y') + self.lon = self._get_coordinate_values(self._lon, 'X') + self.time_bnds = self._time_bnds + + self.write_axis_limits = self.set_write_axis_limits() + + # NetCDF attributes + self.global_attrs = self.__get_global_attributes() # Writing options self.zip_lvl = 0 @@ -313,7 +328,9 @@ class Nes(object): self.variables = self.__get_lazy_variables() else: for var_name in var_list: - del self.variables[var_name] + if self.variables is not None: + if var_name in self.variables: + del self.variables[var_name] return None def keep_vars(self, var_list): @@ -652,7 +669,7 @@ class Nes(object): t_units[4:8], t_units[8:10], t_units[10:12], t_units[13:15], t_units[15:17], t_units[17:-4]) return t_units - def __get_time(self): + def __get_time(self, create_nes=False): """ Get the NetCDF time values @@ -665,8 +682,13 @@ class Nes(object): time = self.variables['time'] else: if self.master: - nc_var = self.netcdf.variables['time'] - time = num2date(nc_var[:], self.__parse_time_unit(nc_var.units), calendar=nc_var.calendar) + if create_nes: + units = "days since 1996-12-31 00:00:00" + calendar = "gregorian" + time = num2date([0.], units=units, calendar=calendar) + else: + nc_var = self.netcdf.variables['time'] + time = num2date(nc_var[:], self.__parse_time_unit(nc_var.units), calendar=nc_var.calendar) time = [aux.replace(second=0, microsecond=0) for aux in time] else: time = None @@ -674,7 +696,7 @@ class Nes(object): self.free_vars('time') return time - def __get_time_bnds(self): + def __get_time_bnds(self, create_nes=False): """ Get the NetCDF time bounds values @@ -687,17 +709,21 @@ class Nes(object): time_bnds = self.variables['time_bnds'] else: if self.master: - if 'time_bnds' in self.netcdf.variables.keys(): - time = self.netcdf.variables['time'] - nc_var = self.netcdf.variables['time_bnds'] - time_bnds = num2date(nc_var[:], self.__parse_time_unit(time.units), calendar=time.calendar).tolist() + if not create_nes: + if 'time_bnds' in self.netcdf.variables.keys(): + time = self.netcdf.variables['time'] + nc_var = self.netcdf.variables['time_bnds'] + time_bnds = num2date(nc_var[:], self.__parse_time_unit(time.units), calendar=time.calendar).tolist() + else: + time_bnds = None else: time_bnds = None else: time_bnds = None time_bnds = self.comm.bcast(time_bnds, root=0) - if 'time_bnds' in self.netcdf.variables.keys(): - self.free_vars('time_bnds') + + self.free_vars('time_bnds') + return time_bnds def _get_coordinate_dimension(self, possible_names): @@ -729,6 +755,7 @@ class Nes(object): nc_var['units'] = '' self.free_vars(dimension_name) + return nc_var def _get_coordinate_values(self, coordinate_info, coordinate_axis): @@ -893,7 +920,7 @@ class Nes(object): self.close() return None - def __get_global_attributes(self): + def __get_global_attributes(self, create_nes=False): """ Read the netcdf global attributes @@ -906,8 +933,9 @@ class Nes(object): if self.is_xarray: gl_attrs = self.dataset.attrs else: - for attrname in self.netcdf.ncattrs(): - gl_attrs[attrname] = getattr(self.netcdf, attrname) + if not create_nes: + for attrname in self.netcdf.ncattrs(): + gl_attrs[attrname] = getattr(self.netcdf, attrname) return gl_attrs # ================================================================================================================== @@ -1039,73 +1067,78 @@ class Nes(object): chunking : bool Indicates if you want to chunk the output netCDF """ - for i, (var_name, var_dict) in enumerate(self.variables.items()): - if var_dict['data'] is not None: - if self.print_info: - print("Rank {0:03d}: Writing {1} var ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) - try: - if not chunking: - var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, - zlib=self.zip_lvl > 0, complevel=self.zip_lvl) - else: - if self.master: - chunk_size = var_dict['data'].shape - else: - chunk_size = None - chunk_size = self.comm.bcast(chunk_size, root=0) - var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, - zlib=self.zip_lvl > 0, complevel=self.zip_lvl, chunksizes=chunk_size) + if self.variables is not None: + for i, (var_name, var_dict) in enumerate(self.variables.items()): + if var_dict['data'] is not None: if self.print_info: - print("Rank {0:03d}: Var {1} created ({2}/{3})".format( - self.rank, var_name, i + 1, len(self.variables))) - if self.size > 1: - var.set_collective(True) + print("Rank {0:03d}: Writing {1} var ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) + try: + if not chunking: + var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, + zlib=self.zip_lvl > 0, complevel=self.zip_lvl) + else: + if self.master: + chunk_size = var_dict['data'].shape + else: + chunk_size = None + chunk_size = self.comm.bcast(chunk_size, root=0) + var = netcdf.createVariable(var_name, var_dict['data'].dtype, ('time', 'lev',) + self._var_dim, + zlib=self.zip_lvl > 0, complevel=self.zip_lvl, chunksizes=chunk_size) if self.print_info: - print("Rank {0:03d}: Var {1} collective ({2}/{3})".format( + print("Rank {0:03d}: Var {1} created ({2}/{3})".format( self.rank, var_name, i + 1, len(self.variables))) + if self.size > 1: + var.set_collective(True) + if self.print_info: + print("Rank {0:03d}: Var {1} collective ({2}/{3})".format( + self.rank, var_name, i + 1, len(self.variables))) + + for att_name, att_value in var_dict.items(): + if att_name == 'data': - for att_name, att_value in var_dict.items(): - if att_name == 'data': - - try: - var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], - self.write_axis_limits['z_min']:self.write_axis_limits['z_max'], - self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], - self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value - except ValueError: - var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], - 0, - self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], - self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value - # msg = "*WARNING* '{0}' variable is a 3D field. Setting it on first (0) layer.".format( - # var_name) - # warn(msg) - except IndexError: - raise IndexError("Different shapes. out_shape={0}, data_shp={1}".format( + try: var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], self.write_axis_limits['z_min']:self.write_axis_limits['z_max'], self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], - self.write_axis_limits['x_min']:self.write_axis_limits['x_max']].shape, - att_value.shape)) - if self.print_info: - print("Rank {0:03d}: Var {1} data ({2}/{3})".format(self.rank, var_name, i + 1, + self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value + except ValueError: + var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], + 0, + self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max']] = att_value + # msg = "*WARNING* '{0}' variable is a 3D field. Setting it on first (0) layer.".format( + # var_name) + # warn(msg) + except IndexError: + raise IndexError("Different shapes. out_shape={0}, data_shp={1}".format( + var[self.write_axis_limits['t_min']:self.write_axis_limits['t_max'], + self.write_axis_limits['z_min']:self.write_axis_limits['z_max'], + self.write_axis_limits['y_min']:self.write_axis_limits['y_max'], + self.write_axis_limits['x_min']:self.write_axis_limits['x_max']].shape, + att_value.shape)) + if self.print_info: + print("Rank {0:03d}: Var {1} data ({2}/{3})".format(self.rank, var_name, i + 1, + len(self.variables))) + elif att_name not in ['chunk_size', 'var_dims', 'dimensions']: + var.setncattr(att_name, att_value) + self._set_var_crs(var) + if self.print_info: + print("Rank {0:03d}: Var {1} completed ({2}/{3})".format(self.rank, var_name, i + 1, len(self.variables))) - elif att_name not in ['chunk_size', 'var_dims', 'dimensions']: - var.setncattr(att_name, att_value) - self._set_var_crs(var) - if self.print_info: - print("Rank {0:03d}: Var {1} completed ({2}/{3})".format(self.rank, var_name, i + 1, - len(self.variables))) - except Exception as e: - print("**ERROR** an error has occurred while writing the '{0}' variable".format(var_name)) - # print("**ERROR** an error hase occurred while writing the '{0}' variable".format(var_name), - # file=sys.stderr) - raise e - else: - msg = 'WARNING!!! ' - msg += 'Variable {0} was not loaded. It will not be written.'.format(var_name) - warnings.warn(msg) + except Exception as e: + print("**ERROR** an error has occurred while writing the '{0}' variable".format(var_name)) + # print("**ERROR** an error hase occurred while writing the '{0}' variable".format(var_name), + # file=sys.stderr) + raise e + else: + msg = 'WARNING!!! ' + msg += 'Variable {0} was not loaded. It will not be written.'.format(var_name) + warnings.warn(msg) + def _create_centroids(self): + """ + Must be implemented on inner class + """ return None def _set_crs(self, netcdf): diff --git a/nes/nc_projections/latlon_nes.py b/nes/nc_projections/latlon_nes.py index d5ec6fb6a92dbca49ff590d1186163cf15c83a95..68e49b9fb3a0fcf25ca6a9279ba120dc9303783a 100644 --- a/nes/nc_projections/latlon_nes.py +++ b/nes/nc_projections/latlon_nes.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from nes.nc_projections.default_nes import Nes - +import numpy as np class LatLonNes(Nes): """ @@ -19,7 +19,8 @@ class LatLonNes(Nes): ('lon',) for a regular latitude-longitude projection. """ def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', - avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None): + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, + **kwargs): """ Initialize the LatLonNes class @@ -46,7 +47,11 @@ class LatLonNes(Nes): super(LatLonNes, self).__init__(comm=comm, path=path, info=info, dataset=dataset, xarray=xarray, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, - first_level=first_level, last_level=last_level) + first_level=first_level, last_level=last_level, create_nes=create_nes, + **kwargs) + + if create_nes: + self._lat, self._lon = self._create_centroids(**kwargs) self._var_dim = ('lat', 'lon') self._lat_dim = ('lat',) @@ -70,6 +75,29 @@ class LatLonNes(Nes): return None + def _create_centroids(self, **kwargs): + + # Calculate center latitudes + lat_c_orig = kwargs['lat_orig'] + (kwargs['inc_lat'] / 2) + self.center_lats = np.linspace( + lat_c_orig, lat_c_orig + (kwargs['inc_lat'] * (kwargs['n_lat'] - 1)), kwargs['n_lat']) + + # Calculate center longitudes + lon_c_orig = kwargs['lon_orig'] + (kwargs['inc_lon'] / 2) + self.center_lons = np.linspace( + lon_c_orig, lon_c_orig + (kwargs['inc_lon'] * (kwargs['n_lon'] - 1)), kwargs['n_lon']) + + return {'data': self.center_lats}, {'data': self.center_lons} + + def _create_bounds(self, **kwargs): + + # This function is not being used + spatial_nv = 2 + boundary_lats = self.create_bounds(self.center_lats, kwargs['inc_lat'], spatial_nv) + boundary_lons = self.create_bounds(self.center_lons, kwargs['inc_lon'], spatial_nv) + + return boundary_lats, boundary_lons + @staticmethod def _set_var_crs(var): """ diff --git a/nes/nc_projections/rotated_nes.py b/nes/nc_projections/rotated_nes.py index 06984059c937fa405d35203c31769edbbf75944a..1a8268e4cbe0c1d88acca35e32f9e875f2879710 100644 --- a/nes/nc_projections/rotated_nes.py +++ b/nes/nc_projections/rotated_nes.py @@ -2,7 +2,7 @@ from nes.nc_projections.default_nes import Nes from cfunits import Units - +import numpy as np class RotatedNes(Nes): """ @@ -31,7 +31,8 @@ class RotatedNes(Nes): ('rlat', 'rlon') for a rotated projection. """ def __init__(self, comm=None, path=None, info=False, dataset=None, xarray=False, parallel_method='Y', - avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None): + avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, create_nes=False, + **kwargs): """ Initialize the RotatedNes class @@ -59,21 +60,26 @@ class RotatedNes(Nes): info=info, dataset=dataset, xarray=xarray, parallel_method=parallel_method, avoid_first_hours=avoid_first_hours, avoid_last_hours=avoid_last_hours, - first_level=first_level, last_level=last_level) + first_level=first_level, last_level=last_level, create_nes=create_nes, + **kwargs) + + if create_nes: + self._rlat, self._rlon = self._create_centroids(**kwargs) - self._rlat = self._get_coordinate_dimension('rlat') - self._rlon = self._get_coordinate_dimension('rlon') + else: + self._rlat = self._get_coordinate_dimension('rlat') + self._rlon = self._get_coordinate_dimension('rlon') - self.rlat = self._get_coordinate_values(self._rlat, 'Y') - self.rlon = self._get_coordinate_values(self._rlon, 'X') + self.rlat = self._get_coordinate_values(self._rlat, 'Y') + self.rlon = self._get_coordinate_values(self._rlon, 'X') - self.projection_data = self.get_projection_data() + self.projection_data = self.get_projection_data(create_nes, **kwargs) self._var_dim = ('rlat', 'rlon') self._lat_dim = ('rlat', 'rlon') self._lon_dim = ('rlat', 'rlon') - def get_projection_data(self): + def get_projection_data(self, create_nes, **kwargs): """ Read the projection data @@ -82,8 +88,18 @@ class RotatedNes(Nes): projection : dict Dictionary with the projection data """ - projection = self.variables['rotated_pole'] - self.free_vars('rotated_pole') + if create_nes: + projection = {'data': None, + 'dimensions': (), + 'grid_mapping_name': 'rotated_latitude_longitude', + 'grid_north_pole_latitude': kwargs['grid_north_pole_latitude'], + 'grid_north_pole_longitude': kwargs['grid_north_pole_longitude'], + } + + else: + projection = self.variables['rotated_pole'] + self.free_vars('rotated_pole') + return projection def _create_dimensions(self, netcdf): @@ -133,6 +149,18 @@ class RotatedNes(Nes): return None + def _create_centroids(self, **kwargs): + + # Calculate center latitudes + self.center_lats = np.linspace(kwargs['south_boundary'], kwargs['south_boundary'] + + (kwargs['inc_rlat'] * (kwargs['n_lat'] - 1)), kwargs['n_lat']) + + # Calculate center longitudes + self.center_lons = np.linspace(kwargs['west_boundary'], kwargs['west_boundary'] + + (kwargs['inc_rlon'] * (kwargs['n_lon'] - 1)), kwargs['n_lon']) + + return {'data': self.center_lats}, {'data': self.center_lons} + @staticmethod def _set_var_crs(var): """ @@ -158,7 +186,7 @@ class RotatedNes(Nes): """ mapping = netcdf.createVariable('rotated_pole', 'c') - mapping.grid_mapping_name = 'rotated_latitude_longitude' + mapping.grid_mapping_name = self.projection_data['grid_mapping_name'] mapping.grid_north_pole_latitude = self.projection_data['grid_north_pole_latitude'] mapping.grid_north_pole_longitude = self.projection_data['grid_north_pole_longitude']