diff --git a/CHANGELOG.md b/CHANGELOG.md index 172fe41d268a2124cbbab9003441fc11d80f33a9..5017be67964bf562dddc6adc320d69a8489baf10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # NES CHANGELOG ### 1.1.0 -* Release date: 2023/03/01 +* Release date: 2023/03/02 * Changes and new features: * Improve Lat-Lon to Cartesian coordinates method (used in Providentia). * Function to_shapefile() to create shapefiles from a NES object without losing data from the original grid and being able to select the time and level. diff --git a/nes/__init__.py b/nes/__init__.py index 625d722e7839209b9eaeede4a00b786605269ff5..29592c5be7b97ad2f8358ed8dd9ae5a94be88109 100644 --- a/nes/__init__.py +++ b/nes/__init__.py @@ -1,7 +1,7 @@ -__date__ = "2023-01-18" -__version__ = "1.0.1" +__date__ = "2023-03-02" +__version__ = "1.1.0" from .load_nes import open_netcdf, concatenate_netcdfs from .create_nes import create_nes, from_shapefile from .nc_projections import * -from .methods.cell_measures import calculate_geometry_area \ No newline at end of file +from .methods.cell_measures import calculate_geometry_area diff --git a/nes/methods/cell_measures.py b/nes/methods/cell_measures.py index d93000ecfa0e68815209862317311e462fb90efe..2866b8b9b2856898d189300a4131ef6fa0b8cd0d 100644 --- a/nes/methods/cell_measures.py +++ b/nes/methods/cell_measures.py @@ -68,7 +68,7 @@ def calculate_geometry_area(geometry_list, earth_radius_minor_axis=6356752.3142, Parameters ---------- - geometry_list : list + geometry_list : List List with polygon geometries. earth_radius_minor_axis : float Radius of the minor axis of the Earth. @@ -167,7 +167,7 @@ def mod_huiliers_area(cell_corner_lon, cell_corner_lat): sum += tri_area(point_0, point_1, point_2) # Copy to calculate area of next triangle - if i == (spatial_nv - 1): + if i == (spatial_nv - 1): point_1 = deepcopy(point_2) return sum diff --git a/nes/methods/horizontal_interpolation.py b/nes/methods/horizontal_interpolation.py index 5fc29d87aa68f277eb18744349feada904e72291..ec3c2a9dcf76c9ba74ee32144d700581004b38fe 100644 --- a/nes/methods/horizontal_interpolation.py +++ b/nes/methods/horizontal_interpolation.py @@ -526,6 +526,8 @@ def create_area_conservative_weight_matrix(self, dst_nes, wm_path=None, info=Fal Source projection Nes Object. dst_nes : nes.Nes Final projection Nes object. + wm_path : str + Path where write the weight matrix info: bool Indicates if you want to print extra info during the methods process. diff --git a/nes/methods/vertical_interpolation.py b/nes/methods/vertical_interpolation.py index 40d9c8789691c23902992bc30843a7c0151f43c3..620689944d335e1fd1ab3af714c0d22d96ca0cd5 100644 --- a/nes/methods/vertical_interpolation.py +++ b/nes/methods/vertical_interpolation.py @@ -28,13 +28,13 @@ def add_4d_vertical_info(self, info_to_add): def interpolate_vertical(self, new_levels, new_src_vertical=None, kind='linear', extrapolate=None, info=None): """ - Vertical methods method. + Vertical interpolation. Parameters ---------- self : Nes Source Nes object. - new_levels : list + new_levels : List List of new vertical levels. new_src_vertical kind : str diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index fcb4397c662602ced009ed4ba95d1d24d23c083b..a5cdb8af2f83571d2f81aef7e09b4e1cb9f2d313 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -2,11 +2,9 @@ import sys import gc -import os import warnings import numpy as np import pandas as pd -import datetime from xarray import open_dataset from netCDF4 import Dataset, num2date, date2num from mpi4py import MPI @@ -21,7 +19,6 @@ from ..methods import vertical_interpolation from ..methods import horizontal_interpolation from ..methods import cell_measures from ..nes_formats import to_netcdf_cams_ra -from ..nc_projections import * class Nes(object): @@ -36,7 +33,7 @@ class Nes(object): True when rank == 0. size : int Size of the communicator. - print_info : bool + info : bool Indicates if you want to print reading/writing info. is_xarray : bool (Not working) Indicates if you want to use xarray as default. @@ -55,7 +52,7 @@ class Nes(object): The variables are stored in a dictionary with the var_name as key and another dictionary with the information. The information dictionary contains the 'data' key with None (if the variable is not loaded) or the array values and the other keys are the variable attributes or description. - _time : list + _time : List Complete list of original time step values. _lev : dict Vertical level dictionary with the complete 'data' key for all the values and the rest of the attributes. @@ -72,7 +69,7 @@ class Nes(object): write_axis_limits : dict Dictionary with the 4D limits of the rank data to write. t_min, t_max, z_min, z_max, y_min, y_max, x_min and x_max. - time : list + time : List List of time steps of the rank data. lev : dict Vertical levels dictionary with the portion of 'data' corresponding to the rank values. @@ -123,7 +120,7 @@ class Nes(object): Index of the last level to use. None if it is the last. create_nes : bool Indicates if you want to create the object from scratch (True) or through an existing file. - times : list, None + times : List, None List of times to substitute the current ones while creation. kwargs : Projection dependent parameters to create it from scratch @@ -465,7 +462,7 @@ class Nes(object): Parameters ---------- - time_bnds : list + time_bnds : List List with the new time bounds information to be set. """ @@ -608,7 +605,7 @@ class Nes(object): Parameters ---------- - var_list : list, str + var_list : List, str, list List (or single string) of the variables to be loaded. """ @@ -635,7 +632,7 @@ class Nes(object): Parameters ---------- - var_list : list, str + var_list : List, str List (or single string) of the variables to be loaded. """ @@ -808,6 +805,14 @@ class Nes(object): return None def get_idx_intervals(self): + """ + Calculate the index intervals + + Returns + ------- + dict + Dictionary with the index intervals + """ idx = {'idx_t_min': self.get_time_id(self.hours_start, first=True), 'idx_t_max': self.get_time_id(self.hours_end, first=False), 'idx_z_min': self.first_level, @@ -1266,17 +1271,20 @@ class Nes(object): return idx - def get_coordinate_id(self, array, value, axis=0): + @staticmethod + def get_coordinate_id(array, value, axis=0): """ Get the index of the corresponding coordinate value. Parameters ---------- + array : np.array + Array with the coordinate data value : float Coordinate value to search. - min : bool - Indicates if you want to apply the filter as minimum (True) or maximum (False) values - Default: True. + axis : int + Axis where find the value + Default: 0. Returns ------- @@ -1367,7 +1375,7 @@ class Nes(object): Parameter --------- - time: list + time: List Original time. units: str CF compliant time units. @@ -1376,7 +1384,7 @@ class Nes(object): Returns ------- - time: list + time: List CF compliant time. """ @@ -1405,7 +1413,7 @@ class Nes(object): Parameters ---------- - time: str + time: Namespace Original time. Returns @@ -1459,7 +1467,7 @@ class Nes(object): Returns ------- - time : list + time : List List of times (datetime.datetime) of the NetCDF data. """ @@ -1490,7 +1498,7 @@ class Nes(object): Returns ------- - time_bnds : list + time_bnds : List List of time bounds (datetime) of the NetCDF data. """ @@ -1527,9 +1535,9 @@ class Nes(object): Returns ------- - lat_bnds : list + lat_bnds : List List of latitude bounds of the NetCDF data. - lon_bnds : list + lon_bnds : List List of longitude bounds of the NetCDF data. """ @@ -1597,7 +1605,7 @@ class Nes(object): Parameters ---------- - possible_names: list, str + possible_names: List, str, list List (or single string) of the possible names of the coordinate (e.g. ['lat', 'latitude']). Returns @@ -1622,8 +1630,7 @@ class Nes(object): self.free_vars(dimension_name) except KeyError: nc_var = {'data': np.array([0]), - 'units': '' - } + 'units': ''} return nc_var @@ -1822,7 +1829,7 @@ class Nes(object): Parameters ---------- - var_list : list, str, None + var_list : List, str, None List (or single string) of the variables to be loaded. """ @@ -2628,7 +2635,7 @@ class Nes(object): Time stamp to select. lev : int Vertical level to select. - var_list : list, str, None + var_list : List, str, None List (or single string) of the variables to be loaded and saved in the shapefile. """ @@ -2683,7 +2690,7 @@ class Nes(object): """ Add variables data to shapefile. - var_list : list + var_list : List List (or single string) of the variables to be loaded and saved in the shapefile. idx_lev : int Index of vertical level for which the data will be saved in the shapefile. @@ -2706,7 +2713,7 @@ class Nes(object): File or path from where the data will be obtained on the intersection. method : str Overlay method. Accepted values: ['nearest', 'intersection', 'centroid']. - var_list : list, None + var_list : List, None Variables that will be included in the resulting shapefile. """ if isinstance(ext_shp, str): @@ -3054,7 +3061,7 @@ class Nes(object): ---------- self : Nes Source Nes object. - new_levels : list + new_levels : List List of new vertical levels. new_src_vertical kind : str @@ -3119,7 +3126,7 @@ class Nes(object): Parameters ---------- - geometry_list : list + geometry_list : List List with polygon geometries. earth_radius_minor_axis : float Radius of the minor axis of the Earth. diff --git a/nes/nc_projections/latlon_nes.py b/nes/nc_projections/latlon_nes.py index 2dfd51310f4bd56d7f14d9d12f46c2130e15aced..cfcb11dae3619e336abbc0978e18ede3be9bfd2f 100644 --- a/nes/nc_projections/latlon_nes.py +++ b/nes/nc_projections/latlon_nes.py @@ -259,6 +259,8 @@ class LatLonNes(Nes): Parameters ---------- + lat_flip : bool + Indicates if the latitudes have to be flipped path : str Path to the output file. grib_keys : dict @@ -275,14 +277,9 @@ class LatLonNes(Nes): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ - - grid_area = super(LatLonNes, self).calculate_grid_area() + grid_area = super().calculate_grid_area() self.cell_measures['cell_area'] = {'data': grid_area.reshape([self.lon_bnds['data'].shape[0], - self.lon_bnds['data'].shape[0]])} + self.lon_bnds['data'].shape[1]])} return None diff --git a/nes/nc_projections/lcc_nes.py b/nes/nc_projections/lcc_nes.py index ae471d8bb2cdf192ea92ab696e31e549b4041a8c..585acbe11a338e65a382ddb1a3d2c544d4325725 100644 --- a/nes/nc_projections/lcc_nes.py +++ b/nes/nc_projections/lcc_nes.py @@ -527,13 +527,7 @@ class LCCNes(Nes): def calculate_grid_area(self): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ - grid_area = super(LCCNes, self).calculate_grid_area() self.cell_measures['cell_area'] = {'data': grid_area.reshape([self.lon_bnds['data'].shape[0], self.lon_bnds['data'].shape[1]])} diff --git a/nes/nc_projections/mercator_nes.py b/nes/nc_projections/mercator_nes.py index eec3fb0af780c3b7cc6883d68742ae11c1010f80..e6ccf187b683ff7d2f17db3cd7657545b024ace7 100644 --- a/nes/nc_projections/mercator_nes.py +++ b/nes/nc_projections/mercator_nes.py @@ -503,11 +503,6 @@ class MercatorNes(Nes): def calculate_grid_area(self): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ grid_area = super(MercatorNes, self).calculate_grid_area() diff --git a/nes/nc_projections/rotated_nes.py b/nes/nc_projections/rotated_nes.py index 5e7cd32aa5c7976bd26577a4d8895590226aeb74..ac66c76a1825e466119da770def6cff7f33ac414 100644 --- a/nes/nc_projections/rotated_nes.py +++ b/nes/nc_projections/rotated_nes.py @@ -553,11 +553,6 @@ class RotatedNes(Nes): def calculate_grid_area(self): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ grid_area = super(RotatedNes, self).calculate_grid_area() diff --git a/tests/run_scalability_tests_nord3v2.sh b/tests/run_scalability_tests_nord3v2.sh index bece03097051b1c2993a2865da42b8a8db36f411..4e6a8fe63f92342b78e262d19dff9e70cf68cdd1 100644 --- a/tests/run_scalability_tests_nord3v2.sh +++ b/tests/run_scalability_tests_nord3v2.sh @@ -1,38 +1,14 @@ #!/bin/bash -EXPORTPATH="/gpfs/projects/bsc32/models/NES_master" -SRCPATH="/gpfs/projects/bsc32/models/NES_master/tests" +EXPORTPATH="/gpfs/scratch/bsc32/bsc32538/NES_tests/NES" +SRCPATH="/gpfs/scratch/bsc32/bsc32538/NES_tests/NES/tests" module purge module load Python/3.7.4-GCCcore-8.3.0 -module load netcdf4-python/1.5.3-foss-2019b-Python-3.7.4 -module load cfunits/1.8-foss-2019b-Python-3.7.4 -module load xarray/0.17.0-foss-2019b-Python-3.7.4 -module load pandas/1.2.4-foss-2019b-Python-3.7.4 -module load mpi4py/3.0.3-foss-2019b-Python-3.7.4 -module load filelock/3.7.1-foss-2019b-Python-3.7.4 -module load pyproj/2.5.0-foss-2019b-Python-3.7.4 -module load eccodes-python/0.9.5-foss-2019b-Python-3.7.4 -module load geopandas/0.10.2-foss-2019b-Python-3.7.4 -module load Shapely/1.7.1-foss-2019b-Python-3.7.4 +module load NES/1.0.0-nord3-v2-foss-2019b-Python-3.7.4 -#EXE="1.1-test_read_write_projection.py" -#EXE="1.2-test_create_projection.py" -#EXE="2.1-test_spatial_join.py" -#EXE="2.2-test_create_shapefile.py" -#EXE="2.3-test_bounds.py" -#EXE="2.4-test_cell_area.py" -#EXE="3.1-test_vertical_interp.py" -#EXE="3.2-test_horiz_interp_bilinear.py" -#EXE="3.3-test_horiz_interp_conservative.py" -#EXE="4.1-test_daily_stats.py" -# for nprocs in 1 2 4 8 16 -# do -# JOB_ID=`sbatch --ntasks=${nprocs} --qos=debug --exclusive --job-name=NES_${EXE}_${nprocs} --output=./log_NES-${EXE}_nord3v2_${nprocs}_%J.out --error=./log_NES-${EXE}_nord3v2_${nprocs}_%J.err -D . --time=02:00:00 --wrap="export PYTHONPATH=${EXPORTPATH}:${PYTHONPATH}; cd ${SRCPATH}; mpirun --mca mpi_warn_on_fork 0 -np ${nprocs} python ${SRCPATH}/${EXE}"` -# done - -for EXE in "1.1-test_read_write_projection.py" "1.2-test_create_projection.py" "2.1-test_spatial_join.py" "2.2-test_create_shapefile.py" "2.3-test_bounds.py" "2.4-test_cell_area.py" "3.1-test_vertical_interp.py" "3.2-test_horiz_interp_bilinear.py" "3.3-test_horiz_interp_conservative.py" "4.1-test_daily_stats.py" +for EXE in "1.1-test_read_write_projection.py" "1.2-test_create_projection.py" "1.3-test_selecting.py" "2.1-test_spatial_join.py" "2.2-test_create_shapefile.py" "2.3-test_bounds.py" "2.4-test_cell_area.py" "3.1-test_vertical_interp.py" "3.2-test_horiz_interp_bilinear.py" "3.3-test_horiz_interp_conservative.py" "4.1-test_daily_stats.py" do for nprocs in 1 2 4 8 16 do diff --git a/tests/sbatch_test_MN4.cmd b/tests/sbatch_test_MN4.cmd deleted file mode 100644 index 09011158b106e8716aab17388ac4c3e5a26c1569..0000000000000000000000000000000000000000 --- a/tests/sbatch_test_MN4.cmd +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -#SBATCH --qos=debug -#SBATCH -A bsc32 -#SBATCH --cpus-per-task=1 -#SBATCH -n 1 -#SBATCH -t 00:30:00 -#SBATCH -J NES_mn4_tests -#SBATCH --output=log_mn4_NES_%j.out -#SBATCH --error=log_mn4_NES_%j.err -#SBATCH --exclusive - -### ulimit -s 128000 - -module purge -module use /gpfs/projects/bsc32/software/suselinux/11/modules/all - -module load Python/3.7.4-GCCcore-8.3.0 -module load netcdf4-python/1.5.3-foss-2019b-Python-3.7.4 -module load cfunits/1.8-foss-2019b-Python-3.7.4 -module load xarray/0.17.0-foss-2019b-Python-3.7.4 -module load OpenMPI/4.0.5-GCC-8.3.0-mn4 -module load filelock/3.7.1-foss-2019b-Python-3.7.4 -module load eccodes-python/0.9.5-foss-2019b-Python-3.7.4 -module load pyproj/2.5.0-foss-2019b-Python-3.7.4 -module load geopandas/0.8.1-foss-2019b-Python-3.7.4 -module load Shapely/1.7.1-foss-2019b-Python-3.7.4 - -# export PYTHONPATH=/gpfs/projects/bsc32/models/NES_master:${PYTHONPATH} -# cd /gpfs/projects/bsc32/models/NES_master/tests - -export PYTHONPATH=/gpfs/scratch/bsc32/bsc32538/NES_tests/NES:${PYTHONPATH} -cd /gpfs/scratch/bsc32/bsc32538/NES_tests/NES/tests - -mpirun --mca mpi_warn_on_fork 0 -np 1 python test_interp_conservative.py diff --git a/tests/test_bash_mn4.cmd b/tests/test_bash_mn4.cmd index 9dfd27dbb3826faa0805dc9d1a552a5fcf81424f..f8663a1d6f5d79192ab77f1418acb5e596cbe98c 100644 --- a/tests/test_bash_mn4.cmd +++ b/tests/test_bash_mn4.cmd @@ -15,13 +15,22 @@ module purge module use /gpfs/projects/bsc32/software/suselinux/11/modules/all -# TODO change module when installed -module load NES/1.0.0-mn4-foss-2019b-Python-3.7.4 +module load NES/1.1.0-mn4-foss-2019b-Python-3.7.4 +module load OpenMPI/4.0.5-GCC-8.3.0-mn4 cd /gpfs/projects/bsc32/models/NES_master/tests -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.2-test_create_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.3-test_selecting.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.2-test_create_shapefile.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.4-test_cell_area.py -mpirun --mca mpi_warn_on_fork 0 -np 4 python 5.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.1-test_vertical_interp.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.2-test_horiz_interp_bilinear.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 4.1-test_daily_stats.py diff --git a/tests/test_bash_nord3v2.cmd b/tests/test_bash_nord3v2.cmd index e585eab22d231734eae8dc1545e973ab200c8ae0..15f22c9d30942e23ebb094f55a508c8944f70c45 100644 --- a/tests/test_bash_nord3v2.cmd +++ b/tests/test_bash_nord3v2.cmd @@ -14,13 +14,21 @@ module purge -# TODO change module when installed -module load NES/1.0.0-nord3-v2-foss-2019b-Python-3.7.4 +module load NES/1.1.0-nord3-v2-foss-2019b-Python-3.7.4 cd /gpfs/projects/bsc32/models/NES_master/tests -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.2-test_create_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.3-test_selecting.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.2-test_create_shapefile.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.4-test_cell_area.py -mpirun --mca mpi_warn_on_fork 0 -np 4 python 5.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.1-test_vertical_interp.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.2-test_horiz_interp_bilinear.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 4.1-test_daily_stats.py diff --git a/tutorials/Jupyter_bash_nord3v2.cmd b/tutorials/Jupyter_bash_nord3v2.cmd index 4a8edb2c431e0f3030224c9f763f83c9a665fc0d..be19eba5303809cc295c955ab30bc5e4e1d124cc 100644 --- a/tutorials/Jupyter_bash_nord3v2.cmd +++ b/tutorials/Jupyter_bash_nord3v2.cmd @@ -25,8 +25,7 @@ localhost:${port} (prefix w/ https:// if using password) # load modules or conda environments here module load jupyterlab/3.0.9-foss-2019b-Python-3.7.4 -# TODO change module when installed -module load NES/1.0.0-nord3-v2-foss-2019b-Python-3.7.4 +module load NES/1.1.0-nord3-v2-foss-2019b-Python-3.7.4 # DON'T USE ADDRESS BELOW.