From c3ae25f192488ba3fd61670e508c139b9a9fb908 Mon Sep 17 00:00:00 2001 From: ctena Date: Thu, 2 Mar 2023 11:21:13 +0100 Subject: [PATCH] Documenting --- CHANGELOG.md | 2 +- nes/__init__.py | 6 +-- nes/methods/cell_measures.py | 4 +- nes/methods/horizontal_interpolation.py | 2 + nes/methods/vertical_interpolation.py | 4 +- nes/nc_projections/default_nes.py | 67 ++++++++++++++----------- nes/nc_projections/latlon_nes.py | 11 ++-- nes/nc_projections/lcc_nes.py | 6 --- nes/nc_projections/mercator_nes.py | 5 -- nes/nc_projections/rotated_nes.py | 5 -- tests/run_scalability_tests_nord3v2.sh | 32 ++---------- tests/sbatch_test_MN4.cmd | 35 ------------- tests/test_bash_mn4.cmd | 21 +++++--- tests/test_bash_nord3v2.cmd | 20 +++++--- tutorials/Jupyter_bash_nord3v2.cmd | 3 +- 15 files changed, 85 insertions(+), 138 deletions(-) delete mode 100644 tests/sbatch_test_MN4.cmd diff --git a/CHANGELOG.md b/CHANGELOG.md index 172fe41..5017be6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # NES CHANGELOG ### 1.1.0 -* Release date: 2023/03/01 +* Release date: 2023/03/02 * Changes and new features: * Improve Lat-Lon to Cartesian coordinates method (used in Providentia). * Function to_shapefile() to create shapefiles from a NES object without losing data from the original grid and being able to select the time and level. diff --git a/nes/__init__.py b/nes/__init__.py index 625d722..29592c5 100644 --- a/nes/__init__.py +++ b/nes/__init__.py @@ -1,7 +1,7 @@ -__date__ = "2023-01-18" -__version__ = "1.0.1" +__date__ = "2023-03-02" +__version__ = "1.1.0" from .load_nes import open_netcdf, concatenate_netcdfs from .create_nes import create_nes, from_shapefile from .nc_projections import * -from .methods.cell_measures import calculate_geometry_area \ No newline at end of file +from .methods.cell_measures import calculate_geometry_area diff --git a/nes/methods/cell_measures.py b/nes/methods/cell_measures.py index d93000e..2866b8b 100644 --- a/nes/methods/cell_measures.py +++ b/nes/methods/cell_measures.py @@ -68,7 +68,7 @@ def calculate_geometry_area(geometry_list, earth_radius_minor_axis=6356752.3142, Parameters ---------- - geometry_list : list + geometry_list : List List with polygon geometries. earth_radius_minor_axis : float Radius of the minor axis of the Earth. @@ -167,7 +167,7 @@ def mod_huiliers_area(cell_corner_lon, cell_corner_lat): sum += tri_area(point_0, point_1, point_2) # Copy to calculate area of next triangle - if i == (spatial_nv - 1): + if i == (spatial_nv - 1): point_1 = deepcopy(point_2) return sum diff --git a/nes/methods/horizontal_interpolation.py b/nes/methods/horizontal_interpolation.py index 5fc29d8..ec3c2a9 100644 --- a/nes/methods/horizontal_interpolation.py +++ b/nes/methods/horizontal_interpolation.py @@ -526,6 +526,8 @@ def create_area_conservative_weight_matrix(self, dst_nes, wm_path=None, info=Fal Source projection Nes Object. dst_nes : nes.Nes Final projection Nes object. + wm_path : str + Path where write the weight matrix info: bool Indicates if you want to print extra info during the methods process. diff --git a/nes/methods/vertical_interpolation.py b/nes/methods/vertical_interpolation.py index 40d9c87..6206899 100644 --- a/nes/methods/vertical_interpolation.py +++ b/nes/methods/vertical_interpolation.py @@ -28,13 +28,13 @@ def add_4d_vertical_info(self, info_to_add): def interpolate_vertical(self, new_levels, new_src_vertical=None, kind='linear', extrapolate=None, info=None): """ - Vertical methods method. + Vertical interpolation. Parameters ---------- self : Nes Source Nes object. - new_levels : list + new_levels : List List of new vertical levels. new_src_vertical kind : str diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index fcb4397..a5cdb8a 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -2,11 +2,9 @@ import sys import gc -import os import warnings import numpy as np import pandas as pd -import datetime from xarray import open_dataset from netCDF4 import Dataset, num2date, date2num from mpi4py import MPI @@ -21,7 +19,6 @@ from ..methods import vertical_interpolation from ..methods import horizontal_interpolation from ..methods import cell_measures from ..nes_formats import to_netcdf_cams_ra -from ..nc_projections import * class Nes(object): @@ -36,7 +33,7 @@ class Nes(object): True when rank == 0. size : int Size of the communicator. - print_info : bool + info : bool Indicates if you want to print reading/writing info. is_xarray : bool (Not working) Indicates if you want to use xarray as default. @@ -55,7 +52,7 @@ class Nes(object): The variables are stored in a dictionary with the var_name as key and another dictionary with the information. The information dictionary contains the 'data' key with None (if the variable is not loaded) or the array values and the other keys are the variable attributes or description. - _time : list + _time : List Complete list of original time step values. _lev : dict Vertical level dictionary with the complete 'data' key for all the values and the rest of the attributes. @@ -72,7 +69,7 @@ class Nes(object): write_axis_limits : dict Dictionary with the 4D limits of the rank data to write. t_min, t_max, z_min, z_max, y_min, y_max, x_min and x_max. - time : list + time : List List of time steps of the rank data. lev : dict Vertical levels dictionary with the portion of 'data' corresponding to the rank values. @@ -123,7 +120,7 @@ class Nes(object): Index of the last level to use. None if it is the last. create_nes : bool Indicates if you want to create the object from scratch (True) or through an existing file. - times : list, None + times : List, None List of times to substitute the current ones while creation. kwargs : Projection dependent parameters to create it from scratch @@ -465,7 +462,7 @@ class Nes(object): Parameters ---------- - time_bnds : list + time_bnds : List List with the new time bounds information to be set. """ @@ -608,7 +605,7 @@ class Nes(object): Parameters ---------- - var_list : list, str + var_list : List, str, list List (or single string) of the variables to be loaded. """ @@ -635,7 +632,7 @@ class Nes(object): Parameters ---------- - var_list : list, str + var_list : List, str List (or single string) of the variables to be loaded. """ @@ -808,6 +805,14 @@ class Nes(object): return None def get_idx_intervals(self): + """ + Calculate the index intervals + + Returns + ------- + dict + Dictionary with the index intervals + """ idx = {'idx_t_min': self.get_time_id(self.hours_start, first=True), 'idx_t_max': self.get_time_id(self.hours_end, first=False), 'idx_z_min': self.first_level, @@ -1266,17 +1271,20 @@ class Nes(object): return idx - def get_coordinate_id(self, array, value, axis=0): + @staticmethod + def get_coordinate_id(array, value, axis=0): """ Get the index of the corresponding coordinate value. Parameters ---------- + array : np.array + Array with the coordinate data value : float Coordinate value to search. - min : bool - Indicates if you want to apply the filter as minimum (True) or maximum (False) values - Default: True. + axis : int + Axis where find the value + Default: 0. Returns ------- @@ -1367,7 +1375,7 @@ class Nes(object): Parameter --------- - time: list + time: List Original time. units: str CF compliant time units. @@ -1376,7 +1384,7 @@ class Nes(object): Returns ------- - time: list + time: List CF compliant time. """ @@ -1405,7 +1413,7 @@ class Nes(object): Parameters ---------- - time: str + time: Namespace Original time. Returns @@ -1459,7 +1467,7 @@ class Nes(object): Returns ------- - time : list + time : List List of times (datetime.datetime) of the NetCDF data. """ @@ -1490,7 +1498,7 @@ class Nes(object): Returns ------- - time_bnds : list + time_bnds : List List of time bounds (datetime) of the NetCDF data. """ @@ -1527,9 +1535,9 @@ class Nes(object): Returns ------- - lat_bnds : list + lat_bnds : List List of latitude bounds of the NetCDF data. - lon_bnds : list + lon_bnds : List List of longitude bounds of the NetCDF data. """ @@ -1597,7 +1605,7 @@ class Nes(object): Parameters ---------- - possible_names: list, str + possible_names: List, str, list List (or single string) of the possible names of the coordinate (e.g. ['lat', 'latitude']). Returns @@ -1622,8 +1630,7 @@ class Nes(object): self.free_vars(dimension_name) except KeyError: nc_var = {'data': np.array([0]), - 'units': '' - } + 'units': ''} return nc_var @@ -1822,7 +1829,7 @@ class Nes(object): Parameters ---------- - var_list : list, str, None + var_list : List, str, None List (or single string) of the variables to be loaded. """ @@ -2628,7 +2635,7 @@ class Nes(object): Time stamp to select. lev : int Vertical level to select. - var_list : list, str, None + var_list : List, str, None List (or single string) of the variables to be loaded and saved in the shapefile. """ @@ -2683,7 +2690,7 @@ class Nes(object): """ Add variables data to shapefile. - var_list : list + var_list : List List (or single string) of the variables to be loaded and saved in the shapefile. idx_lev : int Index of vertical level for which the data will be saved in the shapefile. @@ -2706,7 +2713,7 @@ class Nes(object): File or path from where the data will be obtained on the intersection. method : str Overlay method. Accepted values: ['nearest', 'intersection', 'centroid']. - var_list : list, None + var_list : List, None Variables that will be included in the resulting shapefile. """ if isinstance(ext_shp, str): @@ -3054,7 +3061,7 @@ class Nes(object): ---------- self : Nes Source Nes object. - new_levels : list + new_levels : List List of new vertical levels. new_src_vertical kind : str @@ -3119,7 +3126,7 @@ class Nes(object): Parameters ---------- - geometry_list : list + geometry_list : List List with polygon geometries. earth_radius_minor_axis : float Radius of the minor axis of the Earth. diff --git a/nes/nc_projections/latlon_nes.py b/nes/nc_projections/latlon_nes.py index 2dfd513..cfcb11d 100644 --- a/nes/nc_projections/latlon_nes.py +++ b/nes/nc_projections/latlon_nes.py @@ -259,6 +259,8 @@ class LatLonNes(Nes): Parameters ---------- + lat_flip : bool + Indicates if the latitudes have to be flipped path : str Path to the output file. grib_keys : dict @@ -275,14 +277,9 @@ class LatLonNes(Nes): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ - - grid_area = super(LatLonNes, self).calculate_grid_area() + grid_area = super().calculate_grid_area() self.cell_measures['cell_area'] = {'data': grid_area.reshape([self.lon_bnds['data'].shape[0], - self.lon_bnds['data'].shape[0]])} + self.lon_bnds['data'].shape[1]])} return None diff --git a/nes/nc_projections/lcc_nes.py b/nes/nc_projections/lcc_nes.py index ae471d8..585acbe 100644 --- a/nes/nc_projections/lcc_nes.py +++ b/nes/nc_projections/lcc_nes.py @@ -527,13 +527,7 @@ class LCCNes(Nes): def calculate_grid_area(self): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ - grid_area = super(LCCNes, self).calculate_grid_area() self.cell_measures['cell_area'] = {'data': grid_area.reshape([self.lon_bnds['data'].shape[0], self.lon_bnds['data'].shape[1]])} diff --git a/nes/nc_projections/mercator_nes.py b/nes/nc_projections/mercator_nes.py index eec3fb0..e6ccf18 100644 --- a/nes/nc_projections/mercator_nes.py +++ b/nes/nc_projections/mercator_nes.py @@ -503,11 +503,6 @@ class MercatorNes(Nes): def calculate_grid_area(self): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ grid_area = super(MercatorNes, self).calculate_grid_area() diff --git a/nes/nc_projections/rotated_nes.py b/nes/nc_projections/rotated_nes.py index 5e7cd32..ac66c76 100644 --- a/nes/nc_projections/rotated_nes.py +++ b/nes/nc_projections/rotated_nes.py @@ -553,11 +553,6 @@ class RotatedNes(Nes): def calculate_grid_area(self): """ Get coordinate bounds and call function to calculate the area of each cell of a grid. - - Parameters - ---------- - self : nes.Nes - Source projection Nes Object. """ grid_area = super(RotatedNes, self).calculate_grid_area() diff --git a/tests/run_scalability_tests_nord3v2.sh b/tests/run_scalability_tests_nord3v2.sh index bece030..4e6a8fe 100644 --- a/tests/run_scalability_tests_nord3v2.sh +++ b/tests/run_scalability_tests_nord3v2.sh @@ -1,38 +1,14 @@ #!/bin/bash -EXPORTPATH="/gpfs/projects/bsc32/models/NES_master" -SRCPATH="/gpfs/projects/bsc32/models/NES_master/tests" +EXPORTPATH="/gpfs/scratch/bsc32/bsc32538/NES_tests/NES" +SRCPATH="/gpfs/scratch/bsc32/bsc32538/NES_tests/NES/tests" module purge module load Python/3.7.4-GCCcore-8.3.0 -module load netcdf4-python/1.5.3-foss-2019b-Python-3.7.4 -module load cfunits/1.8-foss-2019b-Python-3.7.4 -module load xarray/0.17.0-foss-2019b-Python-3.7.4 -module load pandas/1.2.4-foss-2019b-Python-3.7.4 -module load mpi4py/3.0.3-foss-2019b-Python-3.7.4 -module load filelock/3.7.1-foss-2019b-Python-3.7.4 -module load pyproj/2.5.0-foss-2019b-Python-3.7.4 -module load eccodes-python/0.9.5-foss-2019b-Python-3.7.4 -module load geopandas/0.10.2-foss-2019b-Python-3.7.4 -module load Shapely/1.7.1-foss-2019b-Python-3.7.4 +module load NES/1.0.0-nord3-v2-foss-2019b-Python-3.7.4 -#EXE="1.1-test_read_write_projection.py" -#EXE="1.2-test_create_projection.py" -#EXE="2.1-test_spatial_join.py" -#EXE="2.2-test_create_shapefile.py" -#EXE="2.3-test_bounds.py" -#EXE="2.4-test_cell_area.py" -#EXE="3.1-test_vertical_interp.py" -#EXE="3.2-test_horiz_interp_bilinear.py" -#EXE="3.3-test_horiz_interp_conservative.py" -#EXE="4.1-test_daily_stats.py" -# for nprocs in 1 2 4 8 16 -# do -# JOB_ID=`sbatch --ntasks=${nprocs} --qos=debug --exclusive --job-name=NES_${EXE}_${nprocs} --output=./log_NES-${EXE}_nord3v2_${nprocs}_%J.out --error=./log_NES-${EXE}_nord3v2_${nprocs}_%J.err -D . --time=02:00:00 --wrap="export PYTHONPATH=${EXPORTPATH}:${PYTHONPATH}; cd ${SRCPATH}; mpirun --mca mpi_warn_on_fork 0 -np ${nprocs} python ${SRCPATH}/${EXE}"` -# done - -for EXE in "1.1-test_read_write_projection.py" "1.2-test_create_projection.py" "2.1-test_spatial_join.py" "2.2-test_create_shapefile.py" "2.3-test_bounds.py" "2.4-test_cell_area.py" "3.1-test_vertical_interp.py" "3.2-test_horiz_interp_bilinear.py" "3.3-test_horiz_interp_conservative.py" "4.1-test_daily_stats.py" +for EXE in "1.1-test_read_write_projection.py" "1.2-test_create_projection.py" "1.3-test_selecting.py" "2.1-test_spatial_join.py" "2.2-test_create_shapefile.py" "2.3-test_bounds.py" "2.4-test_cell_area.py" "3.1-test_vertical_interp.py" "3.2-test_horiz_interp_bilinear.py" "3.3-test_horiz_interp_conservative.py" "4.1-test_daily_stats.py" do for nprocs in 1 2 4 8 16 do diff --git a/tests/sbatch_test_MN4.cmd b/tests/sbatch_test_MN4.cmd deleted file mode 100644 index 0901115..0000000 --- a/tests/sbatch_test_MN4.cmd +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -#SBATCH --qos=debug -#SBATCH -A bsc32 -#SBATCH --cpus-per-task=1 -#SBATCH -n 1 -#SBATCH -t 00:30:00 -#SBATCH -J NES_mn4_tests -#SBATCH --output=log_mn4_NES_%j.out -#SBATCH --error=log_mn4_NES_%j.err -#SBATCH --exclusive - -### ulimit -s 128000 - -module purge -module use /gpfs/projects/bsc32/software/suselinux/11/modules/all - -module load Python/3.7.4-GCCcore-8.3.0 -module load netcdf4-python/1.5.3-foss-2019b-Python-3.7.4 -module load cfunits/1.8-foss-2019b-Python-3.7.4 -module load xarray/0.17.0-foss-2019b-Python-3.7.4 -module load OpenMPI/4.0.5-GCC-8.3.0-mn4 -module load filelock/3.7.1-foss-2019b-Python-3.7.4 -module load eccodes-python/0.9.5-foss-2019b-Python-3.7.4 -module load pyproj/2.5.0-foss-2019b-Python-3.7.4 -module load geopandas/0.8.1-foss-2019b-Python-3.7.4 -module load Shapely/1.7.1-foss-2019b-Python-3.7.4 - -# export PYTHONPATH=/gpfs/projects/bsc32/models/NES_master:${PYTHONPATH} -# cd /gpfs/projects/bsc32/models/NES_master/tests - -export PYTHONPATH=/gpfs/scratch/bsc32/bsc32538/NES_tests/NES:${PYTHONPATH} -cd /gpfs/scratch/bsc32/bsc32538/NES_tests/NES/tests - -mpirun --mca mpi_warn_on_fork 0 -np 1 python test_interp_conservative.py diff --git a/tests/test_bash_mn4.cmd b/tests/test_bash_mn4.cmd index 9dfd27d..f8663a1 100644 --- a/tests/test_bash_mn4.cmd +++ b/tests/test_bash_mn4.cmd @@ -15,13 +15,22 @@ module purge module use /gpfs/projects/bsc32/software/suselinux/11/modules/all -# TODO change module when installed -module load NES/1.0.0-mn4-foss-2019b-Python-3.7.4 +module load NES/1.1.0-mn4-foss-2019b-Python-3.7.4 +module load OpenMPI/4.0.5-GCC-8.3.0-mn4 cd /gpfs/projects/bsc32/models/NES_master/tests -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.2-test_create_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.3-test_selecting.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.2-test_create_shapefile.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.4-test_cell_area.py -mpirun --mca mpi_warn_on_fork 0 -np 4 python 5.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.1-test_vertical_interp.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.2-test_horiz_interp_bilinear.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 4.1-test_daily_stats.py diff --git a/tests/test_bash_nord3v2.cmd b/tests/test_bash_nord3v2.cmd index e585eab..15f22c9 100644 --- a/tests/test_bash_nord3v2.cmd +++ b/tests/test_bash_nord3v2.cmd @@ -14,13 +14,21 @@ module purge -# TODO change module when installed -module load NES/1.0.0-nord3-v2-foss-2019b-Python-3.7.4 +module load NES/1.1.0-nord3-v2-foss-2019b-Python-3.7.4 cd /gpfs/projects/bsc32/models/NES_master/tests -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py -#mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.1-test_read_write_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.2-test_create_projection.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 1.3-test_selecting.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.1-test_spatial_join.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.2-test_create_shapefile.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.3-test_bounds.py mpirun --mca mpi_warn_on_fork 0 -np 4 python 2.4-test_cell_area.py -mpirun --mca mpi_warn_on_fork 0 -np 4 python 5.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.1-test_vertical_interp.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.2-test_horiz_interp_bilinear.py +mpirun --mca mpi_warn_on_fork 0 -np 4 python 3.3-test_horiz_interp_conservative.py + +mpirun --mca mpi_warn_on_fork 0 -np 4 python 4.1-test_daily_stats.py diff --git a/tutorials/Jupyter_bash_nord3v2.cmd b/tutorials/Jupyter_bash_nord3v2.cmd index 4a8edb2..be19eba 100644 --- a/tutorials/Jupyter_bash_nord3v2.cmd +++ b/tutorials/Jupyter_bash_nord3v2.cmd @@ -25,8 +25,7 @@ localhost:${port} (prefix w/ https:// if using password) # load modules or conda environments here module load jupyterlab/3.0.9-foss-2019b-Python-3.7.4 -# TODO change module when installed -module load NES/1.0.0-nord3-v2-foss-2019b-Python-3.7.4 +module load NES/1.1.0-nord3-v2-foss-2019b-Python-3.7.4 # DON'T USE ADDRESS BELOW. -- GitLab