From af769923d8dc48cadcfa8214e7e16b9adcb9567d Mon Sep 17 00:00:00 2001 From: Alba Vilanova Cortezon Date: Wed, 24 Aug 2022 17:19:02 +0200 Subject: [PATCH] #9 Update tutorials and improve Providentia interpolation --- nes/interpolation/horizontal_interpolation.py | 16 +- nes/nc_projections/default_nes.py | 7 +- nes/nc_projections/points_nes.py | 20 +- nes/nc_projections/points_nes_ghost.py | 11 +- nes/nc_projections/points_nes_providentia.py | 10 +- .../1.1.Read_Write_Regular.ipynb | 7 + .../1.2.Read_Write_Rotated.ipynb | 7 + .../1.3.Read_Write_Points.ipynb | 301 +- .../1.Introduction/1.4.Read_Write_LCC.ipynb | 7 + .../1.5.Read_Write_Mercator.ipynb | 7 + .../4.1.Read_Write_Providentia.ipynb} | 245 +- .../4.2.Horizontal_Interpolation.ipynb | 702 +++++ .../4.3.Providentia_Interpolation.ipynb} | 2496 ++++++++--------- .../4.3.Providentia_Rotated.ipynb | 2473 ---------------- .../4.Providentia/4.4.Providentia_LCC.ipynb | 2098 -------------- 15 files changed, 2130 insertions(+), 6277 deletions(-) rename tutorials/{4.Providentia/4.1.Providentia_Read_Write.ipynb => 1.Introduction/4.1.Read_Write_Providentia.ipynb} (99%) create mode 100644 tutorials/4.Interpolation/4.2.Horizontal_Interpolation.ipynb rename tutorials/{4.Providentia/4.2-Providentia_Regular.ipynb => 4.Interpolation/4.3.Providentia_Interpolation.ipynb} (50%) delete mode 100644 tutorials/4.Providentia/4.3.Providentia_Rotated.ipynb delete mode 100644 tutorials/4.Providentia/4.4.Providentia_LCC.ipynb diff --git a/nes/interpolation/horizontal_interpolation.py b/nes/interpolation/horizontal_interpolation.py index 906ba5a..74dadd9 100644 --- a/nes/interpolation/horizontal_interpolation.py +++ b/nes/interpolation/horizontal_interpolation.py @@ -29,8 +29,10 @@ def interpolate_horizontal(self, dst_grid, weight_matrix_path=None, kind='Neares Kind of horizontal interpolation. Accepted values: ['NearestNeighbour']. n_neighbours : int Used if kind == NearestNeighbour. Number of nearest neighbours to interpolate. Default: 4. - info: bool + info : bool Indicates if you want to print extra info during the interpolation process. + to_providentia : bool + Indicates if we want the interpolated grid in Providentia format. """ # Obtain weight matrix @@ -80,7 +82,7 @@ def interpolate_horizontal(self, dst_grid, weight_matrix_path=None, kind='Neares raise IndexError("Data with vertical levels cannot be interpolated to points") final_dst.variables[var_name]['data'] = final_dst.variables[var_name]['data'].reshape( (src_shape[0], idx.shape[-1])) - if isinstance(dst_grid, nes.PointsNesGHOST): + if isinstance(dst_grid, nes.PointsNesGHOST) and not to_providentia: final_dst = final_dst.to_points() final_dst.global_attrs = self.global_attrs @@ -91,12 +93,10 @@ def interpolate_horizontal(self, dst_grid, weight_matrix_path=None, kind='Neares if isinstance(final_dst, nes.PointsNes): model_centre_lat, model_centre_lon = self.create_providentia_exp_centre_coordinates() grid_edge_lat, grid_edge_lon = self.create_providentia_exp_grid_edge_coordinates() - final_dst.to_providentia(model_centre_lon=model_centre_lon, - model_centre_lat=model_centre_lat, - grid_edge_lon=grid_edge_lon, - grid_edge_lat=grid_edge_lat, - ) - final_dst.load() + final_dst = final_dst.to_providentia(model_centre_lon=model_centre_lon, + model_centre_lat=model_centre_lat, + grid_edge_lon=grid_edge_lon, + grid_edge_lat=grid_edge_lat) else: msg = "The final projection must be points to interpolate an experiment and get it in Providentia format." warnings.warn(msg) diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 81c47db..8edaa24 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -2150,7 +2150,7 @@ class Nes(object): self, new_levels, new_src_vertical=new_src_vertical, kind=kind, extrapolate=extrapolate, info=info) def interpolate_horizontal(self, dst_grid, weight_matrix_path=None, kind='NearestNeighbour', n_neighbours=4, - info=False): + info=False, to_providentia=False): """ Horizontal interpolation from the current grid to another one. @@ -2166,7 +2166,10 @@ class Nes(object): Used if kind == NearestNeighbour. Number of nearest neighbours to interpolate. Default: 4. info: bool Indicates if you want to print extra info during the interpolation process. + to_providentia : bool + Indicates if we want the interpolated grid in Providentia format. """ return horizontal_interpolation.interpolate_horizontal( - self, dst_grid, weight_matrix_path=weight_matrix_path, kind=kind, n_neighbours=n_neighbours, info=info) + self, dst_grid, weight_matrix_path=weight_matrix_path, kind=kind, n_neighbours=n_neighbours, info=info, + to_providentia=to_providentia) diff --git a/nes/nc_projections/points_nes.py b/nes/nc_projections/points_nes.py index 5b81e1c..a17e297 100644 --- a/nes/nc_projections/points_nes.py +++ b/nes/nc_projections/points_nes.py @@ -582,7 +582,6 @@ class PointsNes(Nes): points_nes_providentia = PointsNesProvidentia(comm=self.comm, info=self.info, - dataset=self.netcdf, balanced=self.balanced, parallel_method=self.parallel_method, avoid_first_hours=self.hours_start, @@ -598,5 +597,22 @@ class PointsNes(Nes): lat=self.lat['data'], lon=self.lon['data'] ) + + # Convert dimensions (time, lat, lon) to (time, station) for interpolated variables and reshape data + variables = {} + interpolated_variables = deepcopy(self.variables) + for var_name, var_info in interpolated_variables.items(): + print(var_name) + variables[var_name] = {} + if var_info['dimensions'] == ('time', 'lat', 'lon') and len(var_info['data'].shape) == 2: + variables[var_name]['data'] = var_info['data'].reshape((var_info['data'].shape[1], + var_info['data'].shape[0])) + variables[var_name]['dimensions'] = ('station', 'time') + else: + variables[var_name]['data'] = var_info['data'] + variables[var_name]['dimensions'] = var_info['dimensions'] + + # Set variables + points_nes_providentia.variables = variables - return points_nes_providentia \ No newline at end of file + return points_nes_providentia diff --git a/nes/nc_projections/points_nes_ghost.py b/nes/nc_projections/points_nes_ghost.py index 84ceb1f..a90158a 100644 --- a/nes/nc_projections/points_nes_ghost.py +++ b/nes/nc_projections/points_nes_ghost.py @@ -596,12 +596,13 @@ class PointsNesGHOST(PointsNes): lon=self.lon['data'], times=self.time ) - - points_nes.variables = deepcopy(self.variables) - GHOST_version = str(float(np.unique(points_nes.variables['GHOST_version']['data']))) - metadata_variables = self.get_standard_metadata(GHOST_version) - points_nes.free_vars(metadata_variables) + GHOST_version = str(float(np.unique(self.variables['GHOST_version']['data']))) + metadata_variables = self.get_standard_metadata(GHOST_version) + self.free_vars(metadata_variables) + self.free_vars('station') + points_nes.variables = deepcopy(self.variables) + return points_nes def get_standard_metadata(self, GHOST_version): diff --git a/nes/nc_projections/points_nes_providentia.py b/nes/nc_projections/points_nes_providentia.py index 6d44b7f..d45f826 100644 --- a/nes/nc_projections/points_nes_providentia.py +++ b/nes/nc_projections/points_nes_providentia.py @@ -113,14 +113,6 @@ class PointsNesProvidentia(PointsNes): # Set strlen to be None (avoid default strlen inherited from points) self.strlen = None - # Force NES to have a netcdf, despite create_nes=True to read the variables information. - # If we pass the variables with points_nes_providentia.variables = deepcopy(self.variables) inside points_nes.py - # we get wrong the data in the wrong coordinates, i.e. for concentrations in points NES the coordinates are - # (time, station), but for Providentia points NES they are (station, time) as in GHOST - self.netcdf = dataset - self.variables = self._get_lazy_variables() - self.free_vars(['latitude', 'longitude', 'time']) - @staticmethod def new(comm=None, path=None, info=False, dataset=None, xarray=False, create_nes=False, balanced=False, parallel_method='X', avoid_first_hours=0, avoid_last_hours=0, first_level=0, last_level=None, @@ -321,7 +313,7 @@ class PointsNesProvidentia(PointsNes): nc_var = self.netcdf.variables[var_name] var_dims = nc_var.dimensions - # Read data in 1 or 2 dimensions + # Read data in 1, 2 or 3 dimensions if len(var_dims) < 2: data = nc_var[self.read_axis_limits['x_min']:self.read_axis_limits['x_max']] elif len(var_dims) == 2: diff --git a/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb b/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb index f079a46..8042212 100644 --- a/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb +++ b/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb @@ -1908,6 +1908,13 @@ "nessy_2" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reopen with xarray" + ] + }, { "cell_type": "code", "execution_count": 15, diff --git a/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb b/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb index f260511..87ab4d2 100644 --- a/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb +++ b/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb @@ -823,6 +823,13 @@ "nessy_2" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reopen with xarray" + ] + }, { "cell_type": "code", "execution_count": 13, diff --git a/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb b/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb index 2457c7e..582007a 100644 --- a/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb +++ b/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb @@ -418,7 +418,7 @@ " station_end_date (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n", " station_rural_back (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n", " latitude (station) float32 46.81 47.48 ... 53.33 38.88\n", - " station_ozone_classification (station) |S75 b'rural' b'rural' ... b'nan'
    • " ], "text/plain": [ "\n", @@ -710,7 +710,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -1408,7 +1408,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -1808,7 +1808,7 @@ " station_rural_back (station, strlen) object 'n' 'a' 'n' ... '' ''\n", " station_ozone_classification (station, strlen) object 'r' 'u' 'r' ... '' ''\n", "Attributes:\n", - " Conventions: CF-1.7
  • Conventions :
    CF-1.7
    • " ], "text/plain": [ "\n", @@ -2411,7 +2411,7 @@ " source: Surface observations\n", " creator_name: Dene R. Bowdalo\n", " creator_email: dene.bowdalo@bsc.es\n", - " version: 1.4
  • title :
    Surface sulphate data in the EANET network in 2019-11.
    institution :
    Barcelona Supercomputing Center
    source :
    Surface observations
    creator_name :
    Dene R. Bowdalo
    creator_email :
    dene.bowdalo@bsc.es
    version :
    1.4
    • " ], "text/plain": [ "\n", @@ -2637,7 +2637,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -3377,21 +3377,7 @@ "Rank 000: Var MODIS_MCD12C1_v6_modal_LAI_5km data (50/173)\n", "Rank 000: Var MODIS_MCD12C1_v6_modal_LAI_5km completed (50/173)\n", "Rank 000: Writing MODIS_MCD12C1_v6_modal_UMD_land_use_25km var (51/173)\n", - "Rank 000: Var MODIS_MCD12C1_v6_modal_UMD_land_use_25km created (51/173)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes_ghost.py:568: UserWarning: WARNING!!! GHOST datasets cannot be written in parallel yet. Changing to serial mode.\n", - " warnings.warn(msg)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Rank 000: Var MODIS_MCD12C1_v6_modal_UMD_land_use_25km created (51/173)\n", "Rank 000: Var MODIS_MCD12C1_v6_modal_UMD_land_use_25km data (51/173)\n", "Rank 000: Var MODIS_MCD12C1_v6_modal_UMD_land_use_25km completed (51/173)\n", "Rank 000: Writing MODIS_MCD12C1_v6_modal_UMD_land_use_5km var (52/173)\n", @@ -3406,7 +3392,21 @@ "Rank 000: Var NOAA-DMSP-OLS_v4_average_nighttime_stable_lights_5km created (54/173)\n", "Rank 000: Var NOAA-DMSP-OLS_v4_average_nighttime_stable_lights_5km data (54/173)\n", "Rank 000: Var NOAA-DMSP-OLS_v4_average_nighttime_stable_lights_5km completed (54/173)\n", - "Rank 000: Writing NOAA-DMSP-OLS_v4_max_nighttime_stable_lights_25km var (55/173)\n", + "Rank 000: Writing NOAA-DMSP-OLS_v4_max_nighttime_stable_lights_25km var (55/173)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes_ghost.py:570: UserWarning: WARNING!!! GHOST datasets cannot be written in parallel yet. Changing to serial mode.\n", + " warnings.warn(msg)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "Rank 000: Var NOAA-DMSP-OLS_v4_max_nighttime_stable_lights_25km created (55/173)\n", "Rank 000: Var NOAA-DMSP-OLS_v4_max_nighttime_stable_lights_25km data (55/173)\n", "Rank 000: Var NOAA-DMSP-OLS_v4_max_nighttime_stable_lights_25km completed (55/173)\n", @@ -4258,7 +4258,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 26, @@ -4665,7 +4665,7 @@ " creator_name: Dene R. Bowdalo\n", " creator_email: dene.bowdalo@bsc.es\n", " version: 1.4\n", - " Conventions: CF-1.7
  • title :
    Surface sulphate data in the EANET network in 2019-11.
    institution :
    Barcelona Supercomputing Center
    source :
    Surface observations
    creator_name :
    Dene R. Bowdalo
    creator_email :
    dene.bowdalo@bsc.es
    version :
    1.4
    Conventions :
    CF-1.7
    • " ], "text/plain": [ "\n", @@ -4893,7 +4893,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 28, @@ -4914,15 +4914,7 @@ { "data": { "text/plain": [ - "{'station': {'data': masked_array(data=[0., 1., 2.],\n", - " mask=False,\n", - " fill_value=1e+20),\n", - " 'dimensions': ('station',),\n", - " 'units': '',\n", - " 'axis': 'X',\n", - " 'long_name': '',\n", - " 'standard_name': 'station'},\n", - " 'sconcso4': {'data': masked_array(\n", + "{'sconcso4': {'data': masked_array(\n", " data=[[ nan, nan, nan, nan, nan,\n", " nan, nan, 2.31 , 2.31 , 1.12 ,\n", " 1.12 , nan, nan, nan, nan,\n", @@ -4999,39 +4991,20 @@ "Rank 000: Creating points_file_3.nc\n", "Rank 000: NetCDF ready to write\n", "Rank 000: Dimensions done\n", - "Rank 000: Writing station var (1/3)\n", - "**ERROR** an error has occurred while writing the 'station' variable\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "NetCDF: String match to name in use", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnessy_ghost_3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_netcdf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'points_file_3.nc'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/default_nes.py\u001b[0m in \u001b[0;36mto_netcdf\u001b[0;34m(self, path, compression_level, serial, info, chunking)\u001b[0m\n\u001b[1;32m 1786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1787\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1788\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__to_netcdf_py\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunking\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1789\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1790\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mold_info\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/default_nes.py\u001b[0m in \u001b[0;36m__to_netcdf_py\u001b[0;34m(self, path, chunking)\u001b[0m\n\u001b[1;32m 1736\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1737\u001b[0m \u001b[0;31m# Create variables\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1738\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnetcdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunking\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1739\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1740\u001b[0m \u001b[0;31m# Create metadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes.py\u001b[0m in \u001b[0;36m_create_variables\u001b[0;34m(self, netcdf, chunking)\u001b[0m\n\u001b[1;32m 447\u001b[0m \u001b[0;31m# print(\"**ERROR** an error has occurredred while writing the '{0}' variable\".format(var_name),\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 448\u001b[0m \u001b[0;31m# file=sys.stderr)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 450\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 451\u001b[0m \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"WARNING!!! \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes.py\u001b[0m in \u001b[0;36m_create_variables\u001b[0;34m(self, netcdf, chunking)\u001b[0m\n\u001b[1;32m 374\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mchunking\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 375\u001b[0m var = netcdf.createVariable(var_name, var_dtype, var_dims,\n\u001b[0;32m--> 376\u001b[0;31m zlib=self.zip_lvl > 0, complevel=self.zip_lvl)\n\u001b[0m\u001b[1;32m 377\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaster\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32mnetCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Dataset.createVariable\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32mnetCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4.Variable.__init__\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32mnetCDF4/_netCDF4.pyx\u001b[0m in \u001b[0;36mnetCDF4._netCDF4._ensure_nc_success\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: NetCDF: String match to name in use" + "Rank 000: Writing sconcso4 var (1/2)\n", + "Rank 000: Var sconcso4 created (1/2)\n", + "Rank 000: Var sconcso4 data (1/2)\n", + "Rank 000: Var sconcso4 completed (1/2)\n", + "Rank 000: Writing sconcso4_prefiltered_defaultqa var (2/2)\n", + "Rank 000: Var sconcso4_prefiltered_defaultqa created (2/2)\n", + "Rank 000: Var sconcso4_prefiltered_defaultqa data (2/2)\n", + "Rank 000: Var sconcso4_prefiltered_defaultqa completed (2/2)\n" ] } ], "source": [ "nessy_ghost_3.to_netcdf('points_file_3.nc', info=True)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/tutorials/1.Introduction/1.4.Read_Write_LCC.ipynb b/tutorials/1.Introduction/1.4.Read_Write_LCC.ipynb index fe5fa76..e1a4f64 100644 --- a/tutorials/1.Introduction/1.4.Read_Write_LCC.ipynb +++ b/tutorials/1.Introduction/1.4.Read_Write_LCC.ipynb @@ -1259,6 +1259,13 @@ "nessy_2" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reopen with xarray" + ] + }, { "cell_type": "code", "execution_count": 15, diff --git a/tutorials/1.Introduction/1.5.Read_Write_Mercator.ipynb b/tutorials/1.Introduction/1.5.Read_Write_Mercator.ipynb index 9108612..ec31ac6 100644 --- a/tutorials/1.Introduction/1.5.Read_Write_Mercator.ipynb +++ b/tutorials/1.Introduction/1.5.Read_Write_Mercator.ipynb @@ -940,6 +940,13 @@ "nessy_2" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reopen with xarray" + ] + }, { "cell_type": "code", "execution_count": 16, diff --git a/tutorials/4.Providentia/4.1.Providentia_Read_Write.ipynb b/tutorials/1.Introduction/4.1.Read_Write_Providentia.ipynb similarity index 99% rename from tutorials/4.Providentia/4.1.Providentia_Read_Write.ipynb rename to tutorials/1.Introduction/4.1.Read_Write_Providentia.ipynb index 943b833..11985c0 100644 --- a/tutorials/4.Providentia/4.1.Providentia_Read_Write.ipynb +++ b/tutorials/1.Introduction/4.1.Read_Write_Providentia.ipynb @@ -22,7 +22,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Observations dataset" + "## 1. Observations" ] }, { @@ -38,7 +38,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Read" + "### 1.1. Read dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Open with xarray" ] }, { @@ -6305,6 +6312,13 @@ "xr.open_dataset(obs_path)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Open with NES" + ] + }, { "cell_type": "code", "execution_count": 4, @@ -6332,7 +6346,7 @@ "metadata": {}, "outputs": [], "source": [ - "#obs_nes.time" + "obs_nes.time" ] }, { @@ -6877,7 +6891,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Write" + "### 1.2. Write dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Write with NES" ] }, { @@ -7613,6 +7634,13 @@ "obs_nes.to_netcdf('prv_obs_file.nc', info=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Reopen with xarray" + ] + }, { "cell_type": "code", "execution_count": 11, @@ -13894,184 +13922,11 @@ "xr.open_dataset('prv_obs_file.nc')" ] }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(data=[ 201., 17., 124., 1003., 234., 1021., 3075., 1158.,\n", - " 466., 548., 574., 337., 156., 208., 881., 1816.,\n", - " 485., 420., 277., 159., 1733., 39., 555., 3526.,\n", - " 494., 536., 1126., 1024., 796., 2121., 0., 519.,\n", - " 726., 532., 1130., 13., 86., 1197., 97., 869.,\n", - " 6., 981., 2648., nan, 11., 0., 13., 3286.,\n", - " 43., 52., 5., 917., 684., 60., 1260., 122.,\n", - " 1369., 76., 374., 842., 971., 483., 554., 23.,\n", - " 2362., 4., 9., 282., 561., 748., 393., 601.,\n", - " 223., 776., 139., 1760., 770., 225., 2828., 602.,\n", - " 171., 1447., 191., 122., 83., 263., 265., 358.,\n", - " 177., 347., 122., 48., 163., 5., 260., 12.,\n", - " 5., 79., 76., 199., 392., 118., 291., 845.,\n", - " 6., 11., 224., 1775., 4., 1049., 10., 264.,\n", - " 9., 27., 39., 63., 12., 18., 190., 154.,\n", - " 13., 13., 19., 8., 14., 202., 458., 206.,\n", - " 406., 136., 15., 12., 285., 1275., 81., 56.,\n", - " 178., 361., 171., 1588., 6., 195., 780., 9.,\n", - " 338., 397., 7., 170., 51., 181., 36., 221.,\n", - " 234., 107., 514., 766., 1757., 1617., 819., 484.,\n", - " 100., 9., 1691., 3401., nan, 100., 1387., 174.],\n", - " mask=False,\n", - " fill_value=1e+20,\n", - " dtype=float32),\n", - " 'dimensions': ('station',),\n", - " 'standard_name': 'ASTER v3 altitude',\n", - " 'long_name': 'ASTER v3 altitude, relative to EGM96 geoid datum',\n", - " 'units': 'm',\n", - " 'description': 'Altitude from ASTER v3 digital elevation model, relative to EGM96 geoid vertical datum, in metres. The dataset was generated using 1,880,306 Level-1A scenes (taken from the NASA TERRA spacecraft) acquired between March 1, 2000 and November 30, 2013. The ASTER GDEM was created by stacking all individual cloud-masked scene DEMs and non-cloud-masked scene DEMs, then applying various algorithms to remove abnormal data. A statistical approach is not always effective for anomaly removal in areas with a limited number of images. Several existing reference DEMs were used to replace residual anomalies caused by the insufficient number of stacked scenes. In addition to ASTER GDEM, the ASTER Global Water Body Database (ASTWBD) was generated as a by-product to correct elevation values of water body surfaces like sea, rivers, and lakes. The ASTWBD was applied to GDEM to provide proper elevation values for water body surfaces. The sea and lake have a flattened elevation value. The river has a stepped-down elevation value from the upper stream to the lower stream. Native resolution of 1 arc second ~= 30m at the equator.'}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "obs_nes.variables['ASTER_v3_altitude']" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(\n", - " data=[[[2, --, --, ..., --, --, --],\n", - " [2, --, --, ..., --, --, --],\n", - " [2, --, --, ..., --, --, --],\n", - " ...,\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --]],\n", - " \n", - " [[200, --, --, ..., --, --, --],\n", - " [200, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " ...,\n", - " [200, --, --, ..., --, --, --],\n", - " [200, --, --, ..., --, --, --],\n", - " [200, --, --, ..., --, --, --]],\n", - " \n", - " [[0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " ...,\n", - " [60, --, --, ..., --, --, --],\n", - " [60, --, --, ..., --, --, --],\n", - " [60, --, --, ..., --, --, --]],\n", - " \n", - " ...,\n", - " \n", - " [[0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " ...,\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --]],\n", - " \n", - " [[0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " ...,\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --]],\n", - " \n", - " [[0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " ...,\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --],\n", - " [0, --, --, ..., --, --, --]]],\n", - " mask=[[[False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " ...,\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True]],\n", - " \n", - " [[False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " ...,\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True]],\n", - " \n", - " [[False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " ...,\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True]],\n", - " \n", - " ...,\n", - " \n", - " [[False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " ...,\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True]],\n", - " \n", - " [[False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " ...,\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True]],\n", - " \n", - " [[False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " ...,\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True],\n", - " [False, True, True, ..., True, True, True]]],\n", - " fill_value=255,\n", - " dtype=uint8),\n", - " 'dimensions': ('station', 'time', 'N_flag_codes'),\n", - " 'standard_name': 'flags',\n", - " 'long_name': 'data reporter provided standardised flags',\n", - " 'units': '',\n", - " 'description': 'List of associated data flag codes per measurement, indicating the data quality of a specific measurement, provided by the data reporter. Fill value code of 255.'}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "obs_nes.flag" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Experiments dataset" + "## 2. Experiments" ] }, { @@ -14089,7 +13944,14 @@ "tags": [] }, "source": [ - "### Read" + "### 2.1. Read dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Open with xarray" ] }, { @@ -14649,6 +14511,13 @@ "xr.open_dataset(exp_path)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Open with NES" + ] + }, { "cell_type": "code", "execution_count": 16, @@ -14676,7 +14545,7 @@ "metadata": {}, "outputs": [], "source": [ - "#exp_interp_nes.time" + "exp_interp_nes.time" ] }, { @@ -14888,7 +14757,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Write" + "### 2.2. Write dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Write with NES" ] }, { @@ -14926,6 +14802,13 @@ "exp_interp_nes.to_netcdf('prv_exp_file.nc', info=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Reopen with xarray" + ] + }, { "cell_type": "code", "execution_count": 23, diff --git a/tutorials/4.Interpolation/4.2.Horizontal_Interpolation.ipynb b/tutorials/4.Interpolation/4.2.Horizontal_Interpolation.ipynb new file mode 100644 index 0000000..210611c --- /dev/null +++ b/tutorials/4.Interpolation/4.2.Horizontal_Interpolation.ipynb @@ -0,0 +1,702 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to interpolate horizontally" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from nes import *" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Read dataset to interpolate" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "source_path = '/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "source_grid = open_netcdf(path=source_path, info=True)\n", + "source_grid" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': masked_array(\n", + " data=[[16.35033798, 16.43292999, 16.51514626, ..., 16.51514626,\n", + " 16.43292999, 16.35033798],\n", + " [16.52742577, 16.61023903, 16.69267654, ..., 16.69267654,\n", + " 16.61024284, 16.52742577],\n", + " [16.70447159, 16.78750801, 16.87016678, ..., 16.87016678,\n", + " 16.78750992, 16.70447159],\n", + " ...,\n", + " [58.32094955, 58.47268295, 58.62430954, ..., 58.62430954,\n", + " 58.47268295, 58.32094955],\n", + " [58.42628479, 58.57820129, 58.73002625, ..., 58.73002625,\n", + " 58.57820129, 58.42628479],\n", + " [58.53079224, 58.68289948, 58.83491898, ..., 58.83491898,\n", + " 58.68290329, 58.53079224]],\n", + " mask=False,\n", + " fill_value=1e+20),\n", + " 'dimensions': ('rlat', 'rlon'),\n", + " 'units': 'degrees_north',\n", + " 'axis': 'Y',\n", + " 'long_name': 'latitude coordinate',\n", + " 'standard_name': 'latitude'}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "source_grid.lat" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': masked_array(\n", + " data=[[-22.18126488, -22.01667213, -21.85179901, ..., 41.8517952 ,\n", + " 42.01666641, 42.18125916],\n", + " [-22.27817917, -22.11318588, -21.94790459, ..., 41.94789886,\n", + " 42.11317444, 42.27817154],\n", + " [-22.37526703, -22.2098732 , -22.04418945, ..., 42.04418564,\n", + " 42.2098732 , 42.37526321],\n", + " ...,\n", + " [-67.57766724, -67.39706421, -67.21534729, ..., 87.21533966,\n", + " 87.39705658, 87.57765961],\n", + " [-67.90187836, -67.72247314, -67.54193878, ..., 87.54193878,\n", + " 87.72245789, 87.90187073],\n", + " [-68.22803497, -68.04981995, -67.87051392, ..., 87.87050629,\n", + " 88.04981995, 88.22803497]],\n", + " mask=False,\n", + " fill_value=1e+20),\n", + " 'dimensions': ('rlat', 'rlon'),\n", + " 'units': 'degrees_east',\n", + " 'axis': 'X',\n", + " 'long_name': 'longitude coordinate',\n", + " 'standard_name': 'longitude'}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "source_grid.lon" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank 000: Loading O3_all var (1/1)\n", + "Rank 000: Loaded O3_all var ((1, 24, 271, 351))\n" + ] + } + ], + "source": [ + "source_grid.load()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Interpolation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1. With destination grid from file" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Read destination grid" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dst_grid_path = '/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/original_file/MONARCH_d01_2008123100.nc'\n", + "dst_grid = open_netcdf(path=dst_grid_path, info=True)\n", + "dst_grid" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': masked_array(data=[-90., -89., -88., -87., -86., -85., -84., -83., -82.,\n", + " -81., -80., -79., -78., -77., -76., -75., -74., -73.,\n", + " -72., -71., -70., -69., -68., -67., -66., -65., -64.,\n", + " -63., -62., -61., -60., -59., -58., -57., -56., -55.,\n", + " -54., -53., -52., -51., -50., -49., -48., -47., -46.,\n", + " -45., -44., -43., -42., -41., -40., -39., -38., -37.,\n", + " -36., -35., -34., -33., -32., -31., -30., -29., -28.,\n", + " -27., -26., -25., -24., -23., -22., -21., -20., -19.,\n", + " -18., -17., -16., -15., -14., -13., -12., -11., -10.,\n", + " -9., -8., -7., -6., -5., -4., -3., -2., -1.,\n", + " 0., 1., 2., 3., 4., 5., 6., 7., 8.,\n", + " 9., 10., 11., 12., 13., 14., 15., 16., 17.,\n", + " 18., 19., 20., 21., 22., 23., 24., 25., 26.,\n", + " 27., 28., 29., 30., 31., 32., 33., 34., 35.,\n", + " 36., 37., 38., 39., 40., 41., 42., 43., 44.,\n", + " 45., 46., 47., 48., 49., 50., 51., 52., 53.,\n", + " 54., 55., 56., 57., 58., 59., 60., 61., 62.,\n", + " 63., 64., 65., 66., 67., 68., 69., 70., 71.,\n", + " 72., 73., 74., 75., 76., 77., 78., 79., 80.,\n", + " 81., 82., 83., 84., 85., 86., 87., 88., 89.,\n", + " 90.],\n", + " mask=False,\n", + " fill_value=1e+20,\n", + " dtype=float32),\n", + " 'dimensions': ('lat',),\n", + " 'long_name': 'latitude',\n", + " 'units': 'degrees_north',\n", + " 'standard_name': 'grid_latitude'}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dst_grid.lat" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': masked_array(data=[-180. , -178.59375, -177.1875 , -175.78125,\n", + " -174.375 , -172.96875, -171.5625 , -170.15625,\n", + " -168.75 , -167.34375, -165.9375 , -164.53125,\n", + " -163.125 , -161.71875, -160.3125 , -158.90625,\n", + " -157.5 , -156.09375, -154.6875 , -153.28125,\n", + " -151.875 , -150.46875, -149.0625 , -147.65625,\n", + " -146.25 , -144.84375, -143.4375 , -142.03125,\n", + " -140.625 , -139.21875, -137.8125 , -136.40625,\n", + " -135. , -133.59375, -132.1875 , -130.78125,\n", + " -129.375 , -127.96875, -126.5625 , -125.15625,\n", + " -123.75 , -122.34375, -120.9375 , -119.53125,\n", + " -118.125 , -116.71875, -115.3125 , -113.90625,\n", + " -112.5 , -111.09375, -109.6875 , -108.28125,\n", + " -106.875 , -105.46875, -104.0625 , -102.65625,\n", + " -101.25 , -99.84375, -98.4375 , -97.03125,\n", + " -95.625 , -94.21875, -92.8125 , -91.40625,\n", + " -90. , -88.59375, -87.1875 , -85.78125,\n", + " -84.375 , -82.96875, -81.5625 , -80.15625,\n", + " -78.75 , -77.34375, -75.9375 , -74.53125,\n", + " -73.125 , -71.71875, -70.3125 , -68.90625,\n", + " -67.5 , -66.09375, -64.6875 , -63.28125,\n", + " -61.875 , -60.46875, -59.0625 , -57.65625,\n", + " -56.25 , -54.84375, -53.4375 , -52.03125,\n", + " -50.625 , -49.21875, -47.8125 , -46.40625,\n", + " -45. , -43.59375, -42.1875 , -40.78125,\n", + " -39.375 , -37.96875, -36.5625 , -35.15625,\n", + " -33.75 , -32.34375, -30.9375 , -29.53125,\n", + " -28.125 , -26.71875, -25.3125 , -23.90625,\n", + " -22.5 , -21.09375, -19.6875 , -18.28125,\n", + " -16.875 , -15.46875, -14.0625 , -12.65625,\n", + " -11.25 , -9.84375, -8.4375 , -7.03125,\n", + " -5.625 , -4.21875, -2.8125 , -1.40625,\n", + " 0. , 1.40625, 2.8125 , 4.21875,\n", + " 5.625 , 7.03125, 8.4375 , 9.84375,\n", + " 11.25 , 12.65625, 14.0625 , 15.46875,\n", + " 16.875 , 18.28125, 19.6875 , 21.09375,\n", + " 22.5 , 23.90625, 25.3125 , 26.71875,\n", + " 28.125 , 29.53125, 30.9375 , 32.34375,\n", + " 33.75 , 35.15625, 36.5625 , 37.96875,\n", + " 39.375 , 40.78125, 42.1875 , 43.59375,\n", + " 45. , 46.40625, 47.8125 , 49.21875,\n", + " 50.625 , 52.03125, 53.4375 , 54.84375,\n", + " 56.25 , 57.65625, 59.0625 , 60.46875,\n", + " 61.875 , 63.28125, 64.6875 , 66.09375,\n", + " 67.5 , 68.90625, 70.3125 , 71.71875,\n", + " 73.125 , 74.53125, 75.9375 , 77.34375,\n", + " 78.75 , 80.15625, 81.5625 , 82.96875,\n", + " 84.375 , 85.78125, 87.1875 , 88.59375,\n", + " 90. , 91.40625, 92.8125 , 94.21875,\n", + " 95.625 , 97.03125, 98.4375 , 99.84375,\n", + " 101.25 , 102.65625, 104.0625 , 105.46875,\n", + " 106.875 , 108.28125, 109.6875 , 111.09375,\n", + " 112.5 , 113.90625, 115.3125 , 116.71875,\n", + " 118.125 , 119.53125, 120.9375 , 122.34375,\n", + " 123.75 , 125.15625, 126.5625 , 127.96875,\n", + " 129.375 , 130.78125, 132.1875 , 133.59375,\n", + " 135. , 136.40625, 137.8125 , 139.21875,\n", + " 140.625 , 142.03125, 143.4375 , 144.84375,\n", + " 146.25 , 147.65625, 149.0625 , 150.46875,\n", + " 151.875 , 153.28125, 154.6875 , 156.09375,\n", + " 157.5 , 158.90625, 160.3125 , 161.71875,\n", + " 163.125 , 164.53125, 165.9375 , 167.34375,\n", + " 168.75 , 170.15625, 171.5625 , 172.96875,\n", + " 174.375 , 175.78125, 177.1875 , 178.59375,\n", + " 180. ],\n", + " mask=False,\n", + " fill_value=1e+20,\n", + " dtype=float32),\n", + " 'dimensions': ('lon',),\n", + " 'long_name': 'longitude',\n", + " 'units': 'degrees_east',\n", + " 'standard_name': 'longitude'}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dst_grid.lon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Interpolate" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\tO3_all horizontal interpolation\n" + ] + } + ], + "source": [ + "interpolated_source_grid = source_grid.interpolate_horizontal(dst_grid, weight_matrix_path=None, \n", + " kind='NearestNeighbour', n_neighbours=4,\n", + " info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "interpolated_source_grid" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': masked_array(data=[-90., -89., -88., -87., -86., -85., -84., -83., -82.,\n", + " -81., -80., -79., -78., -77., -76., -75., -74., -73.,\n", + " -72., -71., -70., -69., -68., -67., -66., -65., -64.,\n", + " -63., -62., -61., -60., -59., -58., -57., -56., -55.,\n", + " -54., -53., -52., -51., -50., -49., -48., -47., -46.,\n", + " -45., -44., -43., -42., -41., -40., -39., -38., -37.,\n", + " -36., -35., -34., -33., -32., -31., -30., -29., -28.,\n", + " -27., -26., -25., -24., -23., -22., -21., -20., -19.,\n", + " -18., -17., -16., -15., -14., -13., -12., -11., -10.,\n", + " -9., -8., -7., -6., -5., -4., -3., -2., -1.,\n", + " 0., 1., 2., 3., 4., 5., 6., 7., 8.,\n", + " 9., 10., 11., 12., 13., 14., 15., 16., 17.,\n", + " 18., 19., 20., 21., 22., 23., 24., 25., 26.,\n", + " 27., 28., 29., 30., 31., 32., 33., 34., 35.,\n", + " 36., 37., 38., 39., 40., 41., 42., 43., 44.,\n", + " 45., 46., 47., 48., 49., 50., 51., 52., 53.,\n", + " 54., 55., 56., 57., 58., 59., 60., 61., 62.,\n", + " 63., 64., 65., 66., 67., 68., 69., 70., 71.,\n", + " 72., 73., 74., 75., 76., 77., 78., 79., 80.,\n", + " 81., 82., 83., 84., 85., 86., 87., 88., 89.,\n", + " 90.],\n", + " mask=False,\n", + " fill_value=1e+20,\n", + " dtype=float32),\n", + " 'dimensions': ('lat',),\n", + " 'long_name': 'latitude',\n", + " 'units': 'degrees_north',\n", + " 'standard_name': 'grid_latitude'}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "interpolated_source_grid.lat" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': masked_array(data=[-180. , -178.59375, -177.1875 , -175.78125,\n", + " -174.375 , -172.96875, -171.5625 , -170.15625,\n", + " -168.75 , -167.34375, -165.9375 , -164.53125,\n", + " -163.125 , -161.71875, -160.3125 , -158.90625,\n", + " -157.5 , -156.09375, -154.6875 , -153.28125,\n", + " -151.875 , -150.46875, -149.0625 , -147.65625,\n", + " -146.25 , -144.84375, -143.4375 , -142.03125,\n", + " -140.625 , -139.21875, -137.8125 , -136.40625,\n", + " -135. , -133.59375, -132.1875 , -130.78125,\n", + " -129.375 , -127.96875, -126.5625 , -125.15625,\n", + " -123.75 , -122.34375, -120.9375 , -119.53125,\n", + " -118.125 , -116.71875, -115.3125 , -113.90625,\n", + " -112.5 , -111.09375, -109.6875 , -108.28125,\n", + " -106.875 , -105.46875, -104.0625 , -102.65625,\n", + " -101.25 , -99.84375, -98.4375 , -97.03125,\n", + " -95.625 , -94.21875, -92.8125 , -91.40625,\n", + " -90. , -88.59375, -87.1875 , -85.78125,\n", + " -84.375 , -82.96875, -81.5625 , -80.15625,\n", + " -78.75 , -77.34375, -75.9375 , -74.53125,\n", + " -73.125 , -71.71875, -70.3125 , -68.90625,\n", + " -67.5 , -66.09375, -64.6875 , -63.28125,\n", + " -61.875 , -60.46875, -59.0625 , -57.65625,\n", + " -56.25 , -54.84375, -53.4375 , -52.03125,\n", + " -50.625 , -49.21875, -47.8125 , -46.40625,\n", + " -45. , -43.59375, -42.1875 , -40.78125,\n", + " -39.375 , -37.96875, -36.5625 , -35.15625,\n", + " -33.75 , -32.34375, -30.9375 , -29.53125,\n", + " -28.125 , -26.71875, -25.3125 , -23.90625,\n", + " -22.5 , -21.09375, -19.6875 , -18.28125,\n", + " -16.875 , -15.46875, -14.0625 , -12.65625,\n", + " -11.25 , -9.84375, -8.4375 , -7.03125,\n", + " -5.625 , -4.21875, -2.8125 , -1.40625,\n", + " 0. , 1.40625, 2.8125 , 4.21875,\n", + " 5.625 , 7.03125, 8.4375 , 9.84375,\n", + " 11.25 , 12.65625, 14.0625 , 15.46875,\n", + " 16.875 , 18.28125, 19.6875 , 21.09375,\n", + " 22.5 , 23.90625, 25.3125 , 26.71875,\n", + " 28.125 , 29.53125, 30.9375 , 32.34375,\n", + " 33.75 , 35.15625, 36.5625 , 37.96875,\n", + " 39.375 , 40.78125, 42.1875 , 43.59375,\n", + " 45. , 46.40625, 47.8125 , 49.21875,\n", + " 50.625 , 52.03125, 53.4375 , 54.84375,\n", + " 56.25 , 57.65625, 59.0625 , 60.46875,\n", + " 61.875 , 63.28125, 64.6875 , 66.09375,\n", + " 67.5 , 68.90625, 70.3125 , 71.71875,\n", + " 73.125 , 74.53125, 75.9375 , 77.34375,\n", + " 78.75 , 80.15625, 81.5625 , 82.96875,\n", + " 84.375 , 85.78125, 87.1875 , 88.59375,\n", + " 90. , 91.40625, 92.8125 , 94.21875,\n", + " 95.625 , 97.03125, 98.4375 , 99.84375,\n", + " 101.25 , 102.65625, 104.0625 , 105.46875,\n", + " 106.875 , 108.28125, 109.6875 , 111.09375,\n", + " 112.5 , 113.90625, 115.3125 , 116.71875,\n", + " 118.125 , 119.53125, 120.9375 , 122.34375,\n", + " 123.75 , 125.15625, 126.5625 , 127.96875,\n", + " 129.375 , 130.78125, 132.1875 , 133.59375,\n", + " 135. , 136.40625, 137.8125 , 139.21875,\n", + " 140.625 , 142.03125, 143.4375 , 144.84375,\n", + " 146.25 , 147.65625, 149.0625 , 150.46875,\n", + " 151.875 , 153.28125, 154.6875 , 156.09375,\n", + " 157.5 , 158.90625, 160.3125 , 161.71875,\n", + " 163.125 , 164.53125, 165.9375 , 167.34375,\n", + " 168.75 , 170.15625, 171.5625 , 172.96875,\n", + " 174.375 , 175.78125, 177.1875 , 178.59375,\n", + " 180. ],\n", + " mask=False,\n", + " fill_value=1e+20,\n", + " dtype=float32),\n", + " 'dimensions': ('lon',),\n", + " 'long_name': 'longitude',\n", + " 'units': 'degrees_east',\n", + " 'standard_name': 'longitude'}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "interpolated_source_grid.lon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2. Without destination grid from file" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create destination grid" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "lat_orig = 40\n", + "lon_orig = 0\n", + "inc_lat = 0.2\n", + "inc_lon = 0.2\n", + "n_lat = 20\n", + "n_lon = 20\n", + "dst_grid = create_nes(comm=None, info=False, projection='regular',\n", + " lat_orig=lat_orig, lon_orig=lon_orig, inc_lat=inc_lat, inc_lon=inc_lon, \n", + " n_lat=n_lat, n_lon=n_lon)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': array([40.1, 40.3, 40.5, 40.7, 40.9, 41.1, 41.3, 41.5, 41.7, 41.9, 42.1,\n", + " 42.3, 42.5, 42.7, 42.9, 43.1, 43.3, 43.5, 43.7, 43.9])}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dst_grid.lat" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': array([0.1, 0.3, 0.5, 0.7, 0.9, 1.1, 1.3, 1.5, 1.7, 1.9, 2.1, 2.3, 2.5,\n", + " 2.7, 2.9, 3.1, 3.3, 3.5, 3.7, 3.9])}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dst_grid.lon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Interpolate" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\tO3_all horizontal interpolation\n" + ] + } + ], + "source": [ + "interpolated_source_grid = source_grid.interpolate_horizontal(dst_grid, weight_matrix_path=None, \n", + " kind='NearestNeighbour', n_neighbours=4,\n", + " info=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "interpolated_source_grid" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': array([40.1, 40.3, 40.5, 40.7, 40.9, 41.1, 41.3, 41.5, 41.7, 41.9, 42.1,\n", + " 42.3, 42.5, 42.7, 42.9, 43.1, 43.3, 43.5, 43.7, 43.9])}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "interpolated_source_grid.lat" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': array([0.1, 0.3, 0.5, 0.7, 0.9, 1.1, 1.3, 1.5, 1.7, 1.9, 2.1, 2.3, 2.5,\n", + " 2.7, 2.9, 3.1, 3.3, 3.5, 3.7, 3.9])}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "interpolated_source_grid.lon" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tutorials/4.Providentia/4.2-Providentia_Regular.ipynb b/tutorials/4.Interpolation/4.3.Providentia_Interpolation.ipynb similarity index 50% rename from tutorials/4.Providentia/4.2-Providentia_Regular.ipynb rename to tutorials/4.Interpolation/4.3.Providentia_Interpolation.ipynb index 30a31bd..070e36d 100644 --- a/tutorials/4.Providentia/4.2-Providentia_Regular.ipynb +++ b/tutorials/4.Interpolation/4.3.Providentia_Interpolation.ipynb @@ -1,1414 +1,1261 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to interpolate horizontally using Providentia format" + ] + }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "import xarray as xr\n", - "from nes import *" + "from nes import *\n", + "import xarray as xr" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Before \"to_providentia\"" + "## 1. Read dataset to interpolate" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, + "outputs": [], + "source": [ + "source_path = '/esarchive/recon/ecmwf/cams61/cams61_chimere_ph2/eu/hourly/sconco3/sconco3_201804.nc'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:            (station: 175, time: 720)\n",
-       "Coordinates:\n",
-       "  * time               (time) datetime64[ns] 2018-04-01 ... 2018-04-30T23:00:00\n",
-       "Dimensions without coordinates: station\n",
-       "Data variables:\n",
-       "    latitude           (station) float64 -64.24 -54.85 -22.1 ... 21.57 -34.35\n",
-       "    longitude          (station) float64 -56.62 -68.31 -65.6 ... 103.5 18.49\n",
-       "    sconco3            (station, time) float32 ...\n",
-       "    station_reference  (station) object 'AR0001R_UVP' ... 'ZA0001G_UVP'\n",
-       "Attributes:\n",
-       "    title:          Inverse distance weighting (4 neighbours) interpolated ca...\n",
-       "    institution:    Barcelona Supercomputing Center\n",
-       "    source:         Experiment cams61_chimere_ph2\n",
-       "    creator_name:   Dene R. Bowdalo\n",
-       "    creator_email:  dene.bowdalo@bsc.es\n",
-       "    conventions:    CF-1.7\n",
-       "    data_version:   1.0\n",
-       "    history:        Thu Feb 11 10:19:01 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n",
-       "    NCO:            4.7.2
    " - ], "text/plain": [ - "\n", - "Dimensions: (station: 175, time: 720)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2018-04-01 ... 2018-04-30T23:00:00\n", - "Dimensions without coordinates: station\n", - "Data variables:\n", - " latitude (station) float64 ...\n", - " longitude (station) float64 ...\n", - " sconco3 (station, time) float32 ...\n", - " station_reference (station) object ...\n", - "Attributes:\n", - " title: Inverse distance weighting (4 neighbours) interpolated ca...\n", - " institution: Barcelona Supercomputing Center\n", - " source: Experiment cams61_chimere_ph2\n", - " creator_name: Dene R. Bowdalo\n", - " creator_email: dene.bowdalo@bsc.es\n", - " conventions: CF-1.7\n", - " data_version: 1.0\n", - " history: Thu Feb 11 10:19:01 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n", - " NCO: 4.7.2" + "" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "exp_path = '/gpfs/projects/bsc32/AC_cache/recon/exp_interp/1.3.3/cams61_chimere_ph2-eu-000/hourly/sconco3/EBAS/sconco3_201804.nc'\n", - "exp_ds = xr.open_dataset(exp_path).drop(['grid_edge_latitude', 'grid_edge_longitude', \n", - " 'model_centre_latitude', 'model_centre_longitude'])\n", - "exp_ds.to_netcdf('exp_path_interp_points.nc')\n", - "exp_ds" + "source_grid = open_netcdf(path=source_path, info=True)\n", + "source_grid" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 4, "metadata": {}, - "source": [ - "## After \"to_providentia\"" - ] + "outputs": [ + { + "data": { + "text/plain": [ + "{'sconco3': {'data': None,\n", + " 'dimensions': ('time', 'lat', 'lon'),\n", + " 'coordinates': 'lat lon',\n", + " 'grid_mapping': 'crs',\n", + " 'units': 'ppb'}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "source_grid.variables" + ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:                 (grid_edge: 1125, station: 175, model_latitude: 211, model_longitude: 351, time: 720)\n",
-       "Coordinates:\n",
-       "  * time                    (time) datetime64[ns] 2018-04-01 ... 2018-04-30T2...\n",
-       "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n",
-       "Data variables:\n",
-       "    grid_edge_latitude      (grid_edge) float64 29.9 30.1 30.3 ... 29.9 29.9\n",
-       "    grid_edge_longitude     (grid_edge) float64 -25.1 -25.1 ... -24.9 -25.1\n",
-       "    latitude                (station) float64 -64.24 -54.85 ... 21.57 -34.35\n",
-       "    longitude               (station) float64 -56.62 -68.31 ... 103.5 18.49\n",
-       "    model_centre_latitude   (model_latitude, model_longitude) float64 30.0 .....\n",
-       "    model_centre_longitude  (model_latitude, model_longitude) float64 -25.0 ....\n",
-       "    sconco3                 (station, time) float32 ...\n",
-       "    station_reference       (station) object 'AR0001R_UVP' ... 'ZA0001G_UVP'\n",
-       "Attributes:\n",
-       "    title:          Inverse distance weighting (4 neighbours) interpolated ca...\n",
-       "    institution:    Barcelona Supercomputing Center\n",
-       "    source:         Experiment cams61_chimere_ph2\n",
-       "    creator_name:   Dene R. Bowdalo\n",
-       "    creator_email:  dene.bowdalo@bsc.es\n",
-       "    conventions:    CF-1.7\n",
-       "    data_version:   1.0\n",
-       "    history:        Thu Feb 11 10:19:01 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n",
-       "    NCO:            4.7.2
    " - ], - "text/plain": [ - "\n", - "Dimensions: (grid_edge: 1125, station: 175, model_latitude: 211, model_longitude: 351, time: 720)\n", + "\n", + ".xr-dim-list:before {\n", + " content: '(';\n", + "}\n", + "\n", + ".xr-dim-list:after {\n", + " content: ')';\n", + "}\n", + "\n", + ".xr-dim-list li:not(:last-child):after {\n", + " content: ',';\n", + " padding-right: 5px;\n", + "}\n", + "\n", + ".xr-has-index {\n", + " font-weight: bold;\n", + "}\n", + "\n", + ".xr-var-list,\n", + ".xr-var-item {\n", + " display: contents;\n", + "}\n", + "\n", + ".xr-var-item > div,\n", + ".xr-var-item label,\n", + ".xr-var-item > .xr-var-name span {\n", + " background-color: var(--xr-background-color-row-even);\n", + " margin-bottom: 0;\n", + "}\n", + "\n", + ".xr-var-item > .xr-var-name:hover span {\n", + " padding-right: 5px;\n", + "}\n", + "\n", + ".xr-var-list > li:nth-child(odd) > div,\n", + ".xr-var-list > li:nth-child(odd) > label,\n", + ".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n", + " background-color: var(--xr-background-color-row-odd);\n", + "}\n", + "\n", + ".xr-var-name {\n", + " grid-column: 1;\n", + "}\n", + "\n", + ".xr-var-dims {\n", + " grid-column: 2;\n", + "}\n", + "\n", + ".xr-var-dtype {\n", + " grid-column: 3;\n", + " text-align: right;\n", + " color: var(--xr-font-color2);\n", + "}\n", + "\n", + ".xr-var-preview {\n", + " grid-column: 4;\n", + "}\n", + "\n", + ".xr-var-name,\n", + ".xr-var-dims,\n", + ".xr-var-dtype,\n", + ".xr-preview,\n", + ".xr-attrs dt {\n", + " white-space: nowrap;\n", + " overflow: hidden;\n", + " text-overflow: ellipsis;\n", + " padding-right: 10px;\n", + "}\n", + "\n", + ".xr-var-name:hover,\n", + ".xr-var-dims:hover,\n", + ".xr-var-dtype:hover,\n", + ".xr-attrs dt:hover {\n", + " overflow: visible;\n", + " width: auto;\n", + " z-index: 1;\n", + "}\n", + "\n", + ".xr-var-attrs,\n", + ".xr-var-data {\n", + " display: none;\n", + " background-color: var(--xr-background-color) !important;\n", + " padding-bottom: 5px !important;\n", + "}\n", + "\n", + ".xr-var-attrs-in:checked ~ .xr-var-attrs,\n", + ".xr-var-data-in:checked ~ .xr-var-data {\n", + " display: block;\n", + "}\n", + "\n", + ".xr-var-data > table {\n", + " float: right;\n", + "}\n", + "\n", + ".xr-var-name span,\n", + ".xr-var-data,\n", + ".xr-attrs {\n", + " padding-left: 25px !important;\n", + "}\n", + "\n", + ".xr-attrs,\n", + ".xr-var-attrs,\n", + ".xr-var-data {\n", + " grid-column: 1 / -1;\n", + "}\n", + "\n", + "dl.xr-attrs {\n", + " padding: 0;\n", + " margin: 0;\n", + " display: grid;\n", + " grid-template-columns: 125px auto;\n", + "}\n", + "\n", + ".xr-attrs dt,\n", + ".xr-attrs dd {\n", + " padding: 0;\n", + " margin: 0;\n", + " float: left;\n", + " padding-right: 10px;\n", + " width: auto;\n", + "}\n", + "\n", + ".xr-attrs dt {\n", + " font-weight: normal;\n", + " grid-column: 1;\n", + "}\n", + "\n", + ".xr-attrs dt:hover span {\n", + " display: inline-block;\n", + " background: var(--xr-background-color);\n", + " padding-right: 10px;\n", + "}\n", + "\n", + ".xr-attrs dd {\n", + " grid-column: 2;\n", + " white-space: pre-wrap;\n", + " word-break: break-all;\n", + "}\n", + "\n", + ".xr-icon-database,\n", + ".xr-icon-file-text2 {\n", + " display: inline-block;\n", + " vertical-align: middle;\n", + " width: 1em;\n", + " height: 1.5em !important;\n", + " stroke-width: 0;\n", + " stroke: currentColor;\n", + " fill: currentColor;\n", + "}\n", + "
    <xarray.Dataset>\n",
+       "Dimensions:                 (time: 720, station: 168, model_latitude: 211, model_longitude: 351, grid_edge: 1125)\n",
        "Coordinates:\n",
        "  * time                    (time) datetime64[ns] 2018-04-01 ... 2018-04-30T2...\n",
-       "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n",
-       "Data variables:\n",
-       "    grid_edge_latitude      (grid_edge) float64 ...\n",
-       "    grid_edge_longitude     (grid_edge) float64 ...\n",
-       "    latitude                (station) float64 ...\n",
-       "    longitude               (station) float64 ...\n",
-       "    model_centre_latitude   (model_latitude, model_longitude) float64 ...\n",
-       "    model_centre_longitude  (model_latitude, model_longitude) float64 ...\n",
-       "    sconco3                 (station, time) float32 ...\n",
-       "    station_reference       (station) object ...\n",
-       "Attributes:\n",
-       "    title:          Inverse distance weighting (4 neighbours) interpolated ca...\n",
-       "    institution:    Barcelona Supercomputing Center\n",
-       "    source:         Experiment cams61_chimere_ph2\n",
-       "    creator_name:   Dene R. Bowdalo\n",
-       "    creator_email:  dene.bowdalo@bsc.es\n",
-       "    conventions:    CF-1.7\n",
-       "    data_version:   1.0\n",
-       "    history:        Thu Feb 11 10:19:01 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n",
-       "    NCO:            4.7.2"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "xr.open_dataset(exp_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Procedure"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Experiment before interpolation"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "var_ds_before_path = \"/esarchive/recon/ecmwf/cams61/cams61_chimere_ph2/eu/hourly/sconco3/sconco3_201804.nc\"\n",
-    "var_ds_before = open_netcdf(var_ds_before_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "var_ds_before.load()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       ""
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "var_ds_before"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Experiment after interpolation"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/esarchive/scratch/avilanova/software/NES/nes/load_nes.py:69: UserWarning: Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\n",
-      "  warnings.warn(\"Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\")\n"
-     ]
-    }
-   ],
-   "source": [
-    "var_ds_after_path = 'exp_path_interp_points.nc'\n",
-    "var_ds_after = open_netcdf(var_ds_after_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
+       "  * station                 (station) float64 0.0 1.0 2.0 ... 165.0 166.0 167.0\n",
+       "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n",
+       "Data variables:\n",
+       "    lat                     (station) float64 -64.24 -54.85 ... 21.57 -34.35\n",
+       "    lon                     (station) float64 -56.62 -68.31 ... 103.5 18.49\n",
+       "    model_centre_longitude  (model_latitude, model_longitude) float64 -25.0 ....\n",
+       "    model_centre_latitude   (model_latitude, model_longitude) float64 30.0 .....\n",
+       "    grid_edge_longitude     (grid_edge) float64 -25.1 -25.1 ... -24.9 -25.1\n",
+       "    grid_edge_latitude      (grid_edge) float64 29.9 30.1 30.3 ... 29.9 29.9\n",
+       "    sconco3                 (station, time) float64 ...\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.7
    " + ], "text/plain": [ - "" + "\n", + "Dimensions: (time: 720, station: 168, model_latitude: 211, model_longitude: 351, grid_edge: 1125)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2018-04-01 ... 2018-04-30T2...\n", + " * station (station) float64 0.0 1.0 2.0 ... 165.0 166.0 167.0\n", + "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n", + "Data variables:\n", + " lat (station) float64 ...\n", + " lon (station) float64 ...\n", + " model_centre_longitude (model_latitude, model_longitude) float64 ...\n", + " model_centre_latitude (model_latitude, model_longitude) float64 ...\n", + " grid_edge_longitude (grid_edge) float64 ...\n", + " grid_edge_latitude (grid_edge) float64 ...\n", + " sconco3 (station, time) float64 ...\n", + "Attributes:\n", + " Conventions: CF-1.7" ] }, - "execution_count": 8, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "var_ds_after" + "xr.open_dataset('interpolated_source_grid.nc')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Convert to Providentia format" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "model_centre_lat, model_centre_lon = var_ds_before.create_providentia_exp_centre_coordinates()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "grid_edge_lat, grid_edge_lon = var_ds_before.create_providentia_exp_grid_edge_coordinates()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after = var_ds_after.to_providentia(model_centre_lon, model_centre_lat, grid_edge_lon, grid_edge_lat)\n", - "var_ds_after" + "### From Providentia IT" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ - "var_ds_after.load()" + "tool_interpolated_grid_path = '/gpfs/projects/bsc32/AC_cache/recon/exp_interp/1.3.3/cams61_chimere_ph2-eu-000/hourly/sconco3/EBAS/sconco3_201804.nc'" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 20, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "{'data': masked_array(\n", - " data=[[30. , 30. , 30. , ..., 30. , 30. , 30. ],\n", - " [30.2, 30.2, 30.2, ..., 30.2, 30.2, 30.2],\n", - " [30.4, 30.4, 30.4, ..., 30.4, 30.4, 30.4],\n", - " ...,\n", - " [71.6, 71.6, 71.6, ..., 71.6, 71.6, 71.6],\n", - " [71.8, 71.8, 71.8, ..., 71.8, 71.8, 71.8],\n", - " [72. , 72. , 72. , ..., 72. , 72. , 72. ]],\n", - " mask=False,\n", - " fill_value=1e+20)}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.model_centre_lat" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ + "name": "stderr", + "output_type": "stream", + "text": [ + "/esarchive/scratch/avilanova/software/NES/nes/load_nes.py:69: UserWarning: Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\n", + " warnings.warn(\"Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\")\n" + ] + }, { "data": { "text/plain": [ - "{'data': masked_array(\n", - " data=[[-25. , -24.8, -24.6, ..., 44.6, 44.8, 45. ],\n", - " [-25. , -24.8, -24.6, ..., 44.6, 44.8, 45. ],\n", - " [-25. , -24.8, -24.6, ..., 44.6, 44.8, 45. ],\n", - " ...,\n", - " [-25. , -24.8, -24.6, ..., 44.6, 44.8, 45. ],\n", - " [-25. , -24.8, -24.6, ..., 44.6, 44.8, 45. ],\n", - " [-25. , -24.8, -24.6, ..., 44.6, 44.8, 45. ]],\n", - " mask=False,\n", - " fill_value=1e+20)}" + "" ] }, - "execution_count": 14, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "var_ds_after.model_centre_lon" + "tool_interpolated_grid = open_netcdf(path=tool_interpolated_grid_path, info=True)\n", + "tool_interpolated_grid" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 21, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "{'data': masked_array(data=[29.9, 30.1, 30.3, ..., 29.9, 29.9, 29.9],\n", - " mask=False,\n", - " fill_value=1e+20)}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank 000: Loading sconco3 var (1/2)\n", + "Rank 000: Loaded sconco3 var ((175, 720))\n", + "Rank 000: Loading station_reference var (2/2)\n", + "Rank 000: Loaded station_reference var ((175,))\n" + ] } ], "source": [ - "var_ds_after.grid_edge_lat" + "tool_interpolated_grid.load()" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'data': masked_array(data=[-25.1, -25.1, -25.1, ..., -24.7, -24.9, -25.1],\n", - " mask=False,\n", - " fill_value=1e+20)}" + "masked_array(\n", + " data=[[nan, nan, nan, ..., nan, nan, nan],\n", + " [nan, nan, nan, ..., nan, nan, nan],\n", + " [nan, nan, nan, ..., nan, nan, nan],\n", + " ...,\n", + " [nan, nan, nan, ..., nan, nan, nan],\n", + " [nan, nan, nan, ..., nan, nan, nan],\n", + " [nan, nan, nan, ..., nan, nan, nan]],\n", + " mask=False,\n", + " fill_value=1e+20,\n", + " dtype=float32)" ] }, - "execution_count": 16, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "var_ds_after.grid_edge_lon" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Rank 000: Creating test_1.nc\n", - "Rank 000: NetCDF ready to write\n", - "Rank 000: Dimensions done\n", - "Rank 000: Writing sconco3 var (1/2)\n", - "Rank 000: Var sconco3 created (1/2)\n", - "Rank 000: Var sconco3 data (1/2)\n", - "Rank 000: Var sconco3 completed (1/2)\n", - "Rank 000: Writing station_reference var (2/2)\n", - "Rank 000: Var station_reference created (2/2)\n", - "Rank 000: Var station_reference data (2/2)\n", - "Rank 000: Var station_reference completed (2/2)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes_providentia.py:587: UserWarning: WARNING!!! Providentia datasets cannot be written in parallel yet. Changing to serial mode.\n", - " warnings.warn(msg)\n" - ] - } - ], - "source": [ - "var_ds_after.to_netcdf('test_1.nc', info=True)" + "tool_interpolated_grid.variables['sconco3']['data']" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -1766,39 +1613,33 @@ " fill: currentColor;\n", "}\n", "
    <xarray.Dataset>\n",
-       "Dimensions:                 (time: 720, station: 175, model_latitude: 211, model_longitude: 351, grid_edge: 1125)\n",
+       "Dimensions:                 (grid_edge: 1125, station: 175, model_latitude: 211, model_longitude: 351, time: 720)\n",
        "Coordinates:\n",
        "  * time                    (time) datetime64[ns] 2018-04-01 ... 2018-04-30T2...\n",
-       "  * station                 (station) float64 0.0 1.0 2.0 ... 172.0 173.0 174.0\n",
-       "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n",
+       "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n",
        "Data variables:\n",
-       "    lat                     (station) float64 -64.24 -54.85 ... 21.57 -34.35\n",
-       "    lon                     (station) float64 -56.62 -68.31 ... 103.5 18.49\n",
-       "    model_centre_longitude  (model_latitude, model_longitude) float64 -25.0 ....\n",
-       "    model_centre_latitude   (model_latitude, model_longitude) float64 30.0 .....\n",
-       "    grid_edge_longitude     (grid_edge) float64 -25.1 -25.1 ... -24.9 -25.1\n",
        "    grid_edge_latitude      (grid_edge) float64 29.9 30.1 30.3 ... 29.9 29.9\n",
+       "    grid_edge_longitude     (grid_edge) float64 -25.1 -25.1 ... -24.9 -25.1\n",
+       "    latitude                (station) float64 -64.24 -54.85 ... 21.57 -34.35\n",
+       "    longitude               (station) float64 -56.62 -68.31 ... 103.5 18.49\n",
+       "    model_centre_latitude   (model_latitude, model_longitude) float64 30.0 .....\n",
+       "    model_centre_longitude  (model_latitude, model_longitude) float64 -25.0 ....\n",
        "    sconco3                 (station, time) float32 ...\n",
        "    station_reference       (station) object 'AR0001R_UVP' ... 'ZA0001G_UVP'\n",
        "Attributes:\n",
-       "    Conventions:  CF-1.7
  • title :
    Inverse distance weighting (4 neighbours) interpolated cams61_chimere_ph2 experiment data for the component sconco3 with reference to the measurement stations in the EBAS network in 2018-04.
    institution :
    Barcelona Supercomputing Center
    source :
    Experiment cams61_chimere_ph2
    creator_name :
    Dene R. Bowdalo
    creator_email :
    dene.bowdalo@bsc.es
    conventions :
    CF-1.7
    data_version :
    1.0
    history :
    Thu Feb 11 10:19:01 2021: ncks -O --dfl_lvl 1 /gpfs/projects/bsc32/AC_cache/recon/exp_interp/1.3.3/cams61_chimere_ph2-eu-000/hourly/sconco3/EBAS/sconco3_201804.nc /gpfs/projects/bsc32/AC_cache/recon/exp_interp/1.3.3/cams61_chimere_ph2-eu-000/hourly/sconco3/EBAS/sconco3_201804.nc
    NCO :
    4.7.2
    • " ], "text/plain": [ "\n", - "Dimensions: (time: 720, station: 175, model_latitude: 211, model_longitude: 351, grid_edge: 1125)\n", + "Dimensions: (grid_edge: 1125, station: 175, model_latitude: 211, model_longitude: 351, time: 720)\n", "Coordinates:\n", " * time (time) datetime64[ns] 2018-04-01 ... 2018-04-30T2...\n", - " * station (station) float64 0.0 1.0 2.0 ... 172.0 173.0 174.0\n", - "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n", + "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n", "Data variables:\n", - " lat (station) float64 ...\n", - " lon (station) float64 ...\n", - " model_centre_longitude (model_latitude, model_longitude) float64 ...\n", - " model_centre_latitude (model_latitude, model_longitude) float64 ...\n", - " grid_edge_longitude (grid_edge) float64 ...\n", " grid_edge_latitude (grid_edge) float64 ...\n", + " grid_edge_longitude (grid_edge) float64 ...\n", + " latitude (station) float64 ...\n", + " longitude (station) float64 ...\n", + " model_centre_latitude (model_latitude, model_longitude) float64 ...\n", + " model_centre_longitude (model_latitude, model_longitude) float64 ...\n", " sconco3 (station, time) float32 ...\n", " station_reference (station) object ...\n", "Attributes:\n", - " Conventions: CF-1.7" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "xr.open_dataset('test_1.nc')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Difference between grid model edges from NES and Providentia IT" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([-3.81469725e-07, 3.81469729e-07, -7.62939450e-07, ...,\n", - " -3.81469725e-07, -3.81469725e-07, -3.81469725e-07])" + " title: Inverse distance weighting (4 neighbours) interpolated ca...\n", + " institution: Barcelona Supercomputing Center\n", + " source: Experiment cams61_chimere_ph2\n", + " creator_name: Dene R. Bowdalo\n", + " creator_email: dene.bowdalo@bsc.es\n", + " conventions: CF-1.7\n", + " data_version: 1.0\n", + " history: Thu Feb 11 10:19:01 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n", + " NCO: 4.7.2" ] }, - "execution_count": 19, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "xr.open_dataset(exp_path).grid_edge_latitude.values - var_ds_after.grid_edge_lat['data'].data" + "xr.open_dataset('/gpfs/projects/bsc32/AC_cache/recon/exp_interp/1.3.3/cams61_chimere_ph2-eu-000/hourly/sconco3/EBAS/sconco3_201804.nc')" ] } ], diff --git a/tutorials/4.Providentia/4.3.Providentia_Rotated.ipynb b/tutorials/4.Providentia/4.3.Providentia_Rotated.ipynb deleted file mode 100644 index 43cec44..0000000 --- a/tutorials/4.Providentia/4.3.Providentia_Rotated.ipynb +++ /dev/null @@ -1,2473 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import xarray as xr\n", - "from nes import *" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Before \"to_providentia\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:            (station: 173, time: 744)\n",
-       "Coordinates:\n",
-       "  * time               (time) datetime64[ns] 2018-01-01 ... 2018-01-31T23:00:00\n",
-       "Dimensions without coordinates: station\n",
-       "Data variables:\n",
-       "    latitude           (station) float64 -64.24 -54.85 -22.1 ... 21.57 -34.35\n",
-       "    longitude          (station) float64 -56.62 -68.31 -65.6 ... 103.5 18.49\n",
-       "    sconco3            (station, time) float32 ...\n",
-       "    station_reference  (station) object 'AR0001R_UVP' ... 'ZA0001G_UVP'\n",
-       "Attributes:\n",
-       "    title:          Inverse distance weighting (4 neighbours) interpolated a2...\n",
-       "    institution:    Barcelona Supercomputing Center\n",
-       "    source:         Experiment a2yb\n",
-       "    creator_name:   Dene R. Bowdalo\n",
-       "    creator_email:  dene.bowdalo@bsc.es\n",
-       "    conventions:    CF-1.7\n",
-       "    data_version:   1.0\n",
-       "    history:        Mon Mar  1 13:45:41 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n",
-       "    NCO:            netCDF Operators version 4.9.2 (Homepage = http://nco.sf....
    " - ], - "text/plain": [ - "\n", - "Dimensions: (station: 173, time: 744)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2018-01-01 ... 2018-01-31T23:00:00\n", - "Dimensions without coordinates: station\n", - "Data variables:\n", - " latitude (station) float64 ...\n", - " longitude (station) float64 ...\n", - " sconco3 (station, time) float32 ...\n", - " station_reference (station) object ...\n", - "Attributes:\n", - " title: Inverse distance weighting (4 neighbours) interpolated a2...\n", - " institution: Barcelona Supercomputing Center\n", - " source: Experiment a2yb\n", - " creator_name: Dene R. Bowdalo\n", - " creator_email: dene.bowdalo@bsc.es\n", - " conventions: CF-1.7\n", - " data_version: 1.0\n", - " history: Mon Mar 1 13:45:41 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n", - " NCO: netCDF Operators version 4.9.2 (Homepage = http://nco.sf...." - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Experiment after interpolating with SNES (ideal)\n", - "exp_path = '/esarchive/recon/prov_interp/1.3.3/a2yb-regional-000/hourly/sconco3/EBAS/sconco3_201801.nc'\n", - "exp_ds = xr.open_dataset(exp_path).drop(['grid_edge_latitude', 'grid_edge_longitude', \n", - " 'model_centre_latitude', 'model_centre_longitude'])\n", - "exp_ds.to_netcdf('exp_path_interp_points_rotated.nc')\n", - "exp_ds" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## After \"to_providentia\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:                 (grid_edge: 1245, station: 173, model_latitude: 271, model_longitude: 351, time: 744)\n",
-       "Coordinates:\n",
-       "  * time                    (time) datetime64[ns] 2018-01-01 ... 2018-01-31T2...\n",
-       "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n",
-       "Data variables:\n",
-       "    grid_edge_latitude      (grid_edge) float64 16.22 16.4 16.57 ... 16.3 16.22\n",
-       "    grid_edge_longitude     (grid_edge) float64 -22.21 -22.31 ... -22.05 -22.21\n",
-       "    latitude                (station) float64 -64.24 -54.85 ... 21.57 -34.35\n",
-       "    longitude               (station) float64 -56.62 -68.31 ... 103.5 18.49\n",
-       "    model_centre_latitude   (model_latitude, model_longitude) float64 16.35 ....\n",
-       "    model_centre_longitude  (model_latitude, model_longitude) float64 -22.18 ...\n",
-       "    sconco3                 (station, time) float32 ...\n",
-       "    station_reference       (station) object 'AR0001R_UVP' ... 'ZA0001G_UVP'\n",
-       "Attributes:\n",
-       "    title:          Inverse distance weighting (4 neighbours) interpolated a2...\n",
-       "    institution:    Barcelona Supercomputing Center\n",
-       "    source:         Experiment a2yb\n",
-       "    creator_name:   Dene R. Bowdalo\n",
-       "    creator_email:  dene.bowdalo@bsc.es\n",
-       "    conventions:    CF-1.7\n",
-       "    data_version:   1.0\n",
-       "    history:        Mon Mar  1 13:45:41 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n",
-       "    NCO:            netCDF Operators version 4.9.2 (Homepage = http://nco.sf....
    " - ], - "text/plain": [ - "\n", - "Dimensions: (grid_edge: 1245, station: 173, model_latitude: 271, model_longitude: 351, time: 744)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2018-01-01 ... 2018-01-31T2...\n", - "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n", - "Data variables:\n", - " grid_edge_latitude (grid_edge) float64 ...\n", - " grid_edge_longitude (grid_edge) float64 ...\n", - " latitude (station) float64 ...\n", - " longitude (station) float64 ...\n", - " model_centre_latitude (model_latitude, model_longitude) float64 ...\n", - " model_centre_longitude (model_latitude, model_longitude) float64 ...\n", - " sconco3 (station, time) float32 ...\n", - " station_reference (station) object ...\n", - "Attributes:\n", - " title: Inverse distance weighting (4 neighbours) interpolated a2...\n", - " institution: Barcelona Supercomputing Center\n", - " source: Experiment a2yb\n", - " creator_name: Dene R. Bowdalo\n", - " creator_email: dene.bowdalo@bsc.es\n", - " conventions: CF-1.7\n", - " data_version: 1.0\n", - " history: Mon Mar 1 13:45:41 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n", - " NCO: netCDF Operators version 4.9.2 (Homepage = http://nco.sf...." - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Experiment after applying to_providentia\n", - "xr.open_dataset(exp_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Procedure" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Experiment before interpolation" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# Only one timestep\n", - "var_ds_before_path = '/esarchive/exp/monarch/a2yb/regional/hourly/sconco3/sconco3-000_2018010100.nc'\n", - "var_ds_before = open_netcdf(var_ds_before_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "var_ds_before.load()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_before" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:       (time: 25, rlat: 271, rlon: 351, lev: 1)\n",
-       "Coordinates:\n",
-       "  * time          (time) datetime64[ns] 2018-01-01 ... 2018-01-02\n",
-       "    lat           (rlat, rlon) float32 16.35 16.43 16.52 ... 58.83 58.68 58.53\n",
-       "    lon           (rlat, rlon) float32 -22.18 -22.02 -21.85 ... 88.05 88.23\n",
-       "  * rlat          (rlat) float32 -27.0 -26.8 -26.6 -26.4 ... 26.4 26.6 26.8 27.0\n",
-       "  * rlon          (rlon) float32 -35.0 -34.8 -34.6 -34.4 ... 34.4 34.6 34.8 35.0\n",
-       "  * lev           (lev) float32 0.0\n",
-       "Data variables:\n",
-       "    sconco3       (time, lev, rlat, rlon) float32 ...\n",
-       "    rotated_pole  |S1 b''
    " - ], - "text/plain": [ - "\n", - "Dimensions: (time: 25, rlat: 271, rlon: 351, lev: 1)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2018-01-01 ... 2018-01-02\n", - " lat (rlat, rlon) float32 ...\n", - " lon (rlat, rlon) float32 ...\n", - " * rlat (rlat) float32 -27.0 -26.8 -26.6 -26.4 ... 26.4 26.6 26.8 27.0\n", - " * rlon (rlon) float32 -35.0 -34.8 -34.6 -34.4 ... 34.4 34.6 34.8 35.0\n", - " * lev (lev) float32 0.0\n", - "Data variables:\n", - " sconco3 (time, lev, rlat, rlon) float32 ...\n", - " rotated_pole |S1 ..." - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "xr.open_dataset(var_ds_before_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Experiment after interpolation" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/load_nes.py:69: UserWarning: Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\n", - " warnings.warn(\"Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\")\n" - ] - } - ], - "source": [ - "var_ds_after_path = 'exp_path_interp_points_rotated.nc'\n", - "var_ds_after = open_netcdf(var_ds_after_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Convert to Providentia format" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "model_centre_lat, model_centre_lon = var_ds_before.create_providentia_exp_centre_coordinates()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "grid_edge_lat, grid_edge_lon = var_ds_before.create_providentia_exp_grid_edge_coordinates()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after = var_ds_after.to_providentia(model_centre_lon, model_centre_lat, grid_edge_lon, grid_edge_lat)\n", - "var_ds_after" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "var_ds_after.load()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(\n", - " data=[[16.350338, 16.43293 , 16.515146, ..., 16.515146, 16.43293 ,\n", - " 16.350338],\n", - " [16.527426, 16.610239, 16.692677, ..., 16.692677, 16.610243,\n", - " 16.527426],\n", - " [16.704472, 16.787508, 16.870167, ..., 16.870167, 16.78751 ,\n", - " 16.704472],\n", - " ...,\n", - " [58.32095 , 58.472683, 58.62431 , ..., 58.62431 , 58.472683,\n", - " 58.32095 ],\n", - " [58.426285, 58.5782 , 58.730026, ..., 58.730026, 58.5782 ,\n", - " 58.426285],\n", - " [58.530792, 58.6829 , 58.83492 , ..., 58.83492 , 58.682903,\n", - " 58.530792]],\n", - " mask=False,\n", - " fill_value=1e+20,\n", - " dtype=float32),\n", - " 'dimensions': ('rlat', 'rlon'),\n", - " 'units': 'degrees_north',\n", - " 'axis': 'Y',\n", - " 'long_name': 'latitude coordinate',\n", - " 'standard_name': 'latitude'}" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.model_centre_lat" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(\n", - " data=[[-22.181265, -22.016672, -21.851799, ..., 41.851795, 42.016666,\n", - " 42.18126 ],\n", - " [-22.27818 , -22.113186, -21.947905, ..., 41.9479 , 42.113174,\n", - " 42.27817 ],\n", - " [-22.375267, -22.209873, -22.04419 , ..., 42.044186, 42.209873,\n", - " 42.375263],\n", - " ...,\n", - " [-67.57767 , -67.397064, -67.21535 , ..., 87.21534 , 87.39706 ,\n", - " 87.57766 ],\n", - " [-67.90188 , -67.72247 , -67.54194 , ..., 87.54194 , 87.72246 ,\n", - " 87.90187 ],\n", - " [-68.228035, -68.04982 , -67.870514, ..., 87.87051 , 88.04982 ,\n", - " 88.228035]],\n", - " mask=False,\n", - " fill_value=1e+20,\n", - " dtype=float32),\n", - " 'dimensions': ('rlat', 'rlon'),\n", - " 'units': 'degrees_east',\n", - " 'axis': 'X',\n", - " 'long_name': 'longitude coordinate',\n", - " 'standard_name': 'longitude'}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.model_centre_lon" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(data=[16.220402, 16.397398, 16.574354, ..., 16.38536 ,\n", - " 16.30307 , 16.220402],\n", - " mask=False,\n", - " fill_value=1e+20,\n", - " dtype=float32)}" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.grid_edge_lat" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(data=[-22.214966, -22.312012, -22.40918 , ..., -21.88617 ,\n", - " -22.05072 , -22.214966],\n", - " mask=False,\n", - " fill_value=1e+20,\n", - " dtype=float32)}" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.grid_edge_lon" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Rank 000: Creating test_2.nc\n", - "Rank 000: NetCDF ready to write\n", - "Rank 000: Dimensions done\n", - "Rank 000: Writing sconco3 var (1/2)\n", - "Rank 000: Var sconco3 created (1/2)\n", - "Rank 000: Var sconco3 data (1/2)\n", - "Rank 000: Var sconco3 completed (1/2)\n", - "Rank 000: Writing station_reference var (2/2)\n", - "Rank 000: Var station_reference created (2/2)\n", - "Rank 000: Var station_reference data (2/2)\n", - "Rank 000: Var station_reference completed (2/2)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes_providentia.py:587: UserWarning: WARNING!!! Providentia datasets cannot be written in parallel yet. Changing to serial mode.\n", - " warnings.warn(msg)\n" - ] - } - ], - "source": [ - "var_ds_after.to_netcdf('test_2.nc', info=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:                 (time: 744, station: 173, model_latitude: 271, model_longitude: 351, grid_edge: 1245)\n",
-       "Coordinates:\n",
-       "  * time                    (time) datetime64[ns] 2018-01-01 ... 2018-01-31T2...\n",
-       "  * station                 (station) float64 0.0 1.0 2.0 ... 170.0 171.0 172.0\n",
-       "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n",
-       "Data variables:\n",
-       "    lat                     (station) float64 -64.24 -54.85 ... 21.57 -34.35\n",
-       "    lon                     (station) float64 -56.62 -68.31 ... 103.5 18.49\n",
-       "    model_centre_longitude  (model_latitude, model_longitude) float64 -22.18 ...\n",
-       "    model_centre_latitude   (model_latitude, model_longitude) float64 16.35 ....\n",
-       "    grid_edge_longitude     (grid_edge) float64 -22.21 -22.31 ... -22.05 -22.21\n",
-       "    grid_edge_latitude      (grid_edge) float64 16.22 16.4 16.57 ... 16.3 16.22\n",
-       "    sconco3                 (station, time) float32 ...\n",
-       "    station_reference       (station) object 'AR0001R_UVP' ... 'ZA0001G_UVP'\n",
-       "Attributes:\n",
-       "    Conventions:  CF-1.7
    " - ], - "text/plain": [ - "\n", - "Dimensions: (time: 744, station: 173, model_latitude: 271, model_longitude: 351, grid_edge: 1245)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2018-01-01 ... 2018-01-31T2...\n", - " * station (station) float64 0.0 1.0 2.0 ... 170.0 171.0 172.0\n", - "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n", - "Data variables:\n", - " lat (station) float64 ...\n", - " lon (station) float64 ...\n", - " model_centre_longitude (model_latitude, model_longitude) float64 ...\n", - " model_centre_latitude (model_latitude, model_longitude) float64 ...\n", - " grid_edge_longitude (grid_edge) float64 ...\n", - " grid_edge_latitude (grid_edge) float64 ...\n", - " sconco3 (station, time) float32 ...\n", - " station_reference (station) object ...\n", - "Attributes:\n", - " Conventions: CF-1.7" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "xr.open_dataset('test_2.nc')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Difference between grid model edges from NES and Providentia IT" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([-3.56718087e-06, -1.55645110e-06, -2.37911228e-06, ...,\n", - " -7.87491363e-07, -1.22642964e-06, -3.56718087e-06])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "xr.open_dataset(exp_path).grid_edge_latitude.values - var_ds_after.grid_edge_lat['data'].data" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/tutorials/4.Providentia/4.4.Providentia_LCC.ipynb b/tutorials/4.Providentia/4.4.Providentia_LCC.ipynb deleted file mode 100644 index 6d32aa5..0000000 --- a/tutorials/4.Providentia/4.4.Providentia_LCC.ipynb +++ /dev/null @@ -1,2098 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import xarray as xr\n", - "from nes import *" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Before \"to_providentia\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:            (station: 8643, time: 744)\n",
-       "Coordinates:\n",
-       "  * time               (time) datetime64[ns] 2021-01-01 ... 2021-01-31T23:00:00\n",
-       "Dimensions without coordinates: station\n",
-       "Data variables:\n",
-       "    latitude           (station) float64 42.51 42.52 42.53 ... 40.95 28.09 28.48\n",
-       "    longitude          (station) float64 1.539 1.565 1.717 ... -17.12 -16.26\n",
-       "    sconco3            (station, time) float32 ...\n",
-       "    station_reference  (station) object 'escaldes-engordany' ... 'palmetum'\n",
-       "Attributes:\n",
-       "    title:          Inverse distance weighting (4 neighbours) interpolated b0...\n",
-       "    institution:    Barcelona Supercomputing Center\n",
-       "    source:         Experiment b075\n",
-       "    creator_name:   Dene R. Bowdalo\n",
-       "    creator_email:  dene.bowdalo@bsc.es\n",
-       "    conventions:    CF-1.7\n",
-       "    data_version:   1.0\n",
-       "    history:        Mon Feb 22 18:52:51 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n",
-       "    NCO:            netCDF Operators version 4.9.2 (Homepage = http://nco.sf....
    " - ], - "text/plain": [ - "\n", - "Dimensions: (station: 8643, time: 744)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2021-01-01 ... 2021-01-31T23:00:00\n", - "Dimensions without coordinates: station\n", - "Data variables:\n", - " latitude (station) float64 ...\n", - " longitude (station) float64 ...\n", - " sconco3 (station, time) float32 ...\n", - " station_reference (station) object ...\n", - "Attributes:\n", - " title: Inverse distance weighting (4 neighbours) interpolated b0...\n", - " institution: Barcelona Supercomputing Center\n", - " source: Experiment b075\n", - " creator_name: Dene R. Bowdalo\n", - " creator_email: dene.bowdalo@bsc.es\n", - " conventions: CF-1.7\n", - " data_version: 1.0\n", - " history: Mon Feb 22 18:52:51 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n", - " NCO: netCDF Operators version 4.9.2 (Homepage = http://nco.sf...." - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Experiment after interpolating with SNES (ideal)\n", - "exp_path = '/esarchive/recon/prov_interp/1.3.3/b075-eu-000/hourly/sconco3/*EEA/sconco3_202101.nc'\n", - "exp_ds = xr.open_dataset(exp_path).drop(['grid_edge_latitude', 'grid_edge_longitude', \n", - " 'model_centre_latitude', 'model_centre_longitude'])\n", - "exp_ds.to_netcdf('exp_path_interp_points_lcc.nc')\n", - "exp_ds" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## After \"to_providentia\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:                 (grid_edge: 1753, station: 8643, model_latitude: 398, model_longitude: 478, time: 744)\n",
-       "Coordinates:\n",
-       "  * time                    (time) datetime64[ns] 2021-01-01 ... 2021-01-31T2...\n",
-       "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n",
-       "Data variables:\n",
-       "    grid_edge_latitude      (grid_edge) float64 19.59 19.69 ... 19.61 19.59\n",
-       "    grid_edge_longitude     (grid_edge) float64 -22.33 -22.35 ... -22.23 -22.33\n",
-       "    latitude                (station) float64 42.51 42.52 42.53 ... 28.09 28.48\n",
-       "    longitude               (station) float64 1.539 1.565 ... -17.12 -16.26\n",
-       "    model_centre_latitude   (model_latitude, model_longitude) float64 ...\n",
-       "    model_centre_longitude  (model_latitude, model_longitude) float64 ...\n",
-       "    sconco3                 (station, time) float32 ...\n",
-       "    station_reference       (station) object 'escaldes-engordany' ... 'palmetum'\n",
-       "Attributes:\n",
-       "    title:          Inverse distance weighting (4 neighbours) interpolated b0...\n",
-       "    institution:    Barcelona Supercomputing Center\n",
-       "    source:         Experiment b075\n",
-       "    creator_name:   Dene R. Bowdalo\n",
-       "    creator_email:  dene.bowdalo@bsc.es\n",
-       "    conventions:    CF-1.7\n",
-       "    data_version:   1.0\n",
-       "    history:        Mon Feb 22 18:52:51 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n",
-       "    NCO:            netCDF Operators version 4.9.2 (Homepage = http://nco.sf....
    " - ], - "text/plain": [ - "\n", - "Dimensions: (grid_edge: 1753, station: 8643, model_latitude: 398, model_longitude: 478, time: 744)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2021-01-01 ... 2021-01-31T2...\n", - "Dimensions without coordinates: grid_edge, station, model_latitude, model_longitude\n", - "Data variables:\n", - " grid_edge_latitude (grid_edge) float64 ...\n", - " grid_edge_longitude (grid_edge) float64 ...\n", - " latitude (station) float64 ...\n", - " longitude (station) float64 ...\n", - " model_centre_latitude (model_latitude, model_longitude) float64 ...\n", - " model_centre_longitude (model_latitude, model_longitude) float64 ...\n", - " sconco3 (station, time) float32 ...\n", - " station_reference (station) object ...\n", - "Attributes:\n", - " title: Inverse distance weighting (4 neighbours) interpolated b0...\n", - " institution: Barcelona Supercomputing Center\n", - " source: Experiment b075\n", - " creator_name: Dene R. Bowdalo\n", - " creator_email: dene.bowdalo@bsc.es\n", - " conventions: CF-1.7\n", - " data_version: 1.0\n", - " history: Mon Feb 22 18:52:51 2021: ncks -O --dfl_lvl 1 /gpfs/proje...\n", - " NCO: netCDF Operators version 4.9.2 (Homepage = http://nco.sf...." - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Experiment after applying to_providentia\n", - "xr.open_dataset(exp_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Procedure" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Experiment before interpolation" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# Only one timestep\n", - "var_ds_before_path = '/esarchive/exp/wrf-hermes-cmaq/b075/eu/hourly/sconco3/sconco3_2021010100.nc'\n", - "var_ds_before = open_netcdf(var_ds_before_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "var_ds_before.load()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_before" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:            (time: 48, y: 398, x: 478, lev: 1)\n",
-       "Coordinates:\n",
-       "  * time               (time) datetime64[ns] 2021-01-01 ... 2021-01-02T23:00:00\n",
-       "    lat                (y, x) float32 ...\n",
-       "    lon                (y, x) float32 ...\n",
-       "  * x                  (x) float64 -2.126e+06 -2.114e+06 ... 3.586e+06 3.598e+06\n",
-       "  * y                  (y) float64 -2.067e+06 -2.055e+06 ... 2.685e+06 2.697e+06\n",
-       "  * lev                (lev) float32 0.0\n",
-       "Data variables:\n",
-       "    sconco3            (time, lev, y, x) float32 ...\n",
-       "    Lambert_conformal  int32 -2147483647
    " - ], - "text/plain": [ - "\n", - "Dimensions: (time: 48, y: 398, x: 478, lev: 1)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2021-01-01 ... 2021-01-02T23:00:00\n", - " lat (y, x) float32 ...\n", - " lon (y, x) float32 ...\n", - " * x (x) float64 -2.126e+06 -2.114e+06 ... 3.586e+06 3.598e+06\n", - " * y (y) float64 -2.067e+06 -2.055e+06 ... 2.685e+06 2.697e+06\n", - " * lev (lev) float32 0.0\n", - "Data variables:\n", - " sconco3 (time, lev, y, x) float32 ...\n", - " Lambert_conformal int32 ..." - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "xr.open_dataset(var_ds_before_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Experiment after interpolation" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/load_nes.py:69: UserWarning: Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\n", - " warnings.warn(\"Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\")\n" - ] - } - ], - "source": [ - "var_ds_after_path = 'exp_path_interp_points_lcc.nc'\n", - "var_ds_after = open_netcdf(var_ds_after_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Convert to Providentia format" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "model_centre_lat, model_centre_lon = var_ds_before.create_providentia_exp_centre_coordinates()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "grid_edge_lat, grid_edge_lon = var_ds_before.create_providentia_exp_grid_edge_coordinates()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after = var_ds_after.to_providentia(model_centre_lon, model_centre_lat, grid_edge_lon, grid_edge_lat)\n", - "var_ds_after" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "var_ds_after.load()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(\n", - " data=[[19.706436, 19.728317, 19.750084, ..., 16.264694, 16.22998 ,\n", - " 16.19516 ],\n", - " [19.805984, 19.827904, 19.849716, ..., 16.358276, 16.323502,\n", - " 16.288643],\n", - " [19.905594, 19.927544, 19.949394, ..., 16.45192 , 16.417084,\n", - " 16.382141],\n", - " ...,\n", - " [59.66961 , 59.70968 , 59.74953 , ..., 53.457195, 53.39534 ,\n", - " 53.33333 ],\n", - " [59.76223 , 59.802353, 59.842262, ..., 53.541912, 53.47999 ,\n", - " 53.417908],\n", - " [59.854744, 59.89492 , 59.934883, ..., 53.62653 , 53.56453 ,\n", - " 53.502373]],\n", - " mask=False,\n", - " fill_value=1e+20,\n", - " dtype=float32),\n", - " 'dimensions': ('y', 'x'),\n", - " 'units': 'degrees_north',\n", - " 'axis': 'Y',\n", - " 'long_name': 'latitude coordinate',\n", - " 'standard_name': 'latitude'}" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.model_centre_lat" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(\n", - " data=[[-22.32898 , -22.223236, -22.117432, ..., 28.619202, 28.716614,\n", - " 28.813965],\n", - " [-22.352325, -22.24646 , -22.140533, ..., 28.655334, 28.752869,\n", - " 28.850311],\n", - " [-22.375702, -22.269714, -22.163696, ..., 28.69159 , 28.789185,\n", - " 28.886719],\n", - " ...,\n", - " [-39.438995, -39.25531 , -39.07132 , ..., 53.106964, 53.249176,\n", - " 53.391052],\n", - " [-39.518707, -39.334717, -39.15039 , ..., 53.210876, 53.35321 ,\n", - " 53.49518 ],\n", - " [-39.598724, -39.41443 , -39.229828, ..., 53.315125, 53.45755 ,\n", - " 53.59964 ]],\n", - " mask=False,\n", - " fill_value=1e+20,\n", - " dtype=float32),\n", - " 'dimensions': ('y', 'x'),\n", - " 'units': 'degrees_east',\n", - " 'axis': 'X',\n", - " 'long_name': 'longitude coordinate',\n", - " 'standard_name': 'longitude'}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.model_centre_lon" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(data=[19.60175375, 19.701436 , 19.80117422, ...,\n", - " 19.64562332, 19.62374577, 19.60175375],\n", - " mask=False,\n", - " fill_value=1e+20)}" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.grid_edge_lat" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': masked_array(data=[-22.40084818, -22.4242827 , -22.44777427, ...,\n", - " -22.18916008, -22.29503477, -22.40084818],\n", - " mask=False,\n", - " fill_value=1e+20)}" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.grid_edge_lon" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(8643, 744)" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "var_ds_after.variables['sconco3']['data'].shape" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes_providentia.py:587: UserWarning: WARNING!!! Providentia datasets cannot be written in parallel yet. Changing to serial mode.\n", - " warnings.warn(msg)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Rank 000: Creating test_3.nc\n", - "Rank 000: NetCDF ready to write\n", - "Rank 000: Dimensions done\n", - "Rank 000: Writing sconco3 var (1/2)\n", - "Rank 000: Var sconco3 created (1/2)\n", - "Rank 000: Var sconco3 data (1/2)\n", - "Rank 000: Var sconco3 completed (1/2)\n", - "Rank 000: Writing station_reference var (2/2)\n", - "Rank 000: Var station_reference created (2/2)\n", - "Rank 000: Var station_reference data (2/2)\n", - "Rank 000: Var station_reference completed (2/2)\n" - ] - } - ], - "source": [ - "var_ds_after.to_netcdf('test_3.nc', info=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:                 (time: 744, station: 8643, model_latitude: 398, model_longitude: 478, grid_edge: 1753)\n",
-       "Coordinates:\n",
-       "  * time                    (time) datetime64[ns] 2021-01-01 ... 2021-01-31T2...\n",
-       "  * station                 (station) float64 0.0 1.0 ... 8.641e+03 8.642e+03\n",
-       "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n",
-       "Data variables:\n",
-       "    lat                     (station) float64 42.51 42.52 42.53 ... 28.09 28.48\n",
-       "    lon                     (station) float64 1.539 1.565 ... -17.12 -16.26\n",
-       "    model_centre_longitude  (model_latitude, model_longitude) float64 ...\n",
-       "    model_centre_latitude   (model_latitude, model_longitude) float64 ...\n",
-       "    grid_edge_longitude     (grid_edge) float64 -22.4 -22.42 ... -22.3 -22.4\n",
-       "    grid_edge_latitude      (grid_edge) float64 19.6 19.7 19.8 ... 19.62 19.6\n",
-       "    sconco3                 (station, time) float32 ...\n",
-       "    station_reference       (station) object 'escaldes-engordany' ... 'palmetum'\n",
-       "Attributes:\n",
-       "    Conventions:  CF-1.7
    " - ], - "text/plain": [ - "\n", - "Dimensions: (time: 744, station: 8643, model_latitude: 398, model_longitude: 478, grid_edge: 1753)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2021-01-01 ... 2021-01-31T2...\n", - " * station (station) float64 0.0 1.0 ... 8.641e+03 8.642e+03\n", - "Dimensions without coordinates: model_latitude, model_longitude, grid_edge\n", - "Data variables:\n", - " lat (station) float64 ...\n", - " lon (station) float64 ...\n", - " model_centre_longitude (model_latitude, model_longitude) float64 ...\n", - " model_centre_latitude (model_latitude, model_longitude) float64 ...\n", - " grid_edge_longitude (grid_edge) float64 ...\n", - " grid_edge_latitude (grid_edge) float64 ...\n", - " sconco3 (station, time) float32 ...\n", - " station_reference (station) object ...\n", - "Attributes:\n", - " Conventions: CF-1.7" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "xr.open_dataset('test_3.nc')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Difference between grid model edges from NES and Providentia IT" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([-0.0111262 , -0.01086253, -0.01059908, ..., -0.01117357,\n", - " -0.01114973, -0.0111262 ])" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "xr.open_dataset(exp_path).grid_edge_latitude.values - var_ds_after.grid_edge_lat['data'].data" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} -- GitLab