From fe1eb59f4fb24c1d07f5e2bb541bca2c25f1b0e7 Mon Sep 17 00:00:00 2001 From: Alba Vilanova Cortezon Date: Thu, 19 Jan 2023 17:16:27 +0100 Subject: [PATCH 1/4] Temporal changes on tutorials --- .gitignore | 4 +- nes/create_nes.py | 4 +- nes/nc_projections/default_nes.py | 74 +- .../1.1.Read_Write_Regular.ipynb | 24 +- .../1.2.Read_Write_Rotated.ipynb | 34 +- .../1.3.Read_Write_Points.ipynb | 237 +- .../1.Introduction/1.4.Read_Write_LCC.ipynb | 997 +++---- .../1.5.Read_Write_Mercator.ipynb | 1185 +++++++- .../1.6.Read_Write_Providentia.ipynb | 705 ++--- .../2.Creation/2.7.Create_Mercator.ipynb | 4 +- tutorials/3.Statistics/3.1.Statistics.ipynb | 276 +- .../4.1.Vertical_Interpolation.ipynb | 287 +- .../4.2.Horizontal_Interpolation.ipynb | 372 ++- .../4.3.Providentia_Interpolation.ipynb | 40 +- tutorials/5.Others/5.1.Add_Time_Bounds.ipynb | 2 +- .../5.Others/5.2.Add_Coordinates_Bounds.ipynb | 2 +- .../5.Others/5.3.Create_Shapefiles.ipynb | 109 +- tutorials/5.Others/5.4.Spatial_Join.ipynb | 356 +-- tutorials/5.Others/5.5.Selecting.ipynb | 175 +- .../5.Others/5.6.Interpolation_Test.ipynb | 2507 ++--------------- 20 files changed, 3388 insertions(+), 4006 deletions(-) diff --git a/.gitignore b/.gitignore index e5d46f5..6f3a619 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,7 @@ .idea logs -tests/basic_nes_tests_alba.py -tests/test_bash_nord3v2-alba.cmd notebooks/.ipynb_checkpoints .ipynb_checkpoints nes/__pycache__ nes/nc_projections/__pycache__ -jupyter_notebooks \ No newline at end of file +*.pyc \ No newline at end of file diff --git a/nes/create_nes.py b/nes/create_nes.py index a4e484f..d722ce8 100644 --- a/nes/create_nes.py +++ b/nes/create_nes.py @@ -139,9 +139,9 @@ def from_shapefile(path, method=None, **kwargs): nessy.create_shapefile() # Read shapefile - mask = gpd.read_file(path) + shapefile = gpd.read_file(path) # Make spatial join - nessy.spatial_join(mask, method=method) + nessy.spatial_join(shapefile, method=method) return nessy diff --git a/nes/nc_projections/default_nes.py b/nes/nc_projections/default_nes.py index 5931073..90cda6a 100644 --- a/nes/nc_projections/default_nes.py +++ b/nes/nc_projections/default_nes.py @@ -1672,7 +1672,7 @@ class Nes(object): var_name)) # Missing to nan try: - data[data.mask == True] = np.nan + data[data.shapefile == True] = np.nan except (AttributeError, MaskError, ValueError): pass @@ -2584,48 +2584,58 @@ class Nes(object): return None - def spatial_join(self, mask, method=None): + def spatial_join(self, shapefile, method=None, var_list=None): """ Compute overlay intersection of two GeoPandasDataFrames. Parameters ---------- - mask : GeoPandasDataFrame - File from where the data will be obtained on the intersection. + shapefile : GeoPandasDataFrame, str + File or path from where the data will be obtained on the intersection. method : str - Overlay method. Accepted values: ['nearest', 'intersection', None]. + Overlay method. Accepted values: ['nearest', 'intersection', 'centroid']. + var_list : list, None + """ + + if isinstance(shapefile, str): + shapefile = gpd.read_file(shapefile) + + if self.shapefile is None: + msg = 'Warning: Shapefile does not exist. It will be created now.' + warnings.warn(msg) + self.create_shapefile() - # Nearest centroids to the mask polygons + # Nearest centroids to the shapefile polygons if method == 'nearest': # Make copy - shapefile_aux = deepcopy(self.shapefile) + aux_grid = deepcopy(self.shapefile) - # Get centroids of shapefile to mask - shapefile_aux.geometry = shapefile_aux.centroid + # Get centroids of shapefile to shapefile + aux_grid.geometry = aux_grid.centroid # Calculate spatial joint by distance - shapefile_aux = gpd.sjoin_nearest(shapefile_aux, mask.to_crs(shapefile_aux.crs), distance_col='distance') + aux_grid = gpd.sjoin_nearest(aux_grid, shapefile.to_crs(aux_grid.crs), distance_col='distance') # Get data from closest shapes to centroids - del shapefile_aux['geometry'], shapefile_aux['index_right'] - self.shapefile.loc[shapefile_aux.index, shapefile_aux.columns] = shapefile_aux + del aux_grid['geometry'], aux_grid['index_right'] + self.shapefile.loc[aux_grid.index, aux_grid.columns] = aux_grid - # Intersect the areas of the mask polygons, outside of the mask there will be NaN + # Intersect the areas of the shapefile polygons, outside of the shapefile there will be NaN elif method == 'intersection': # Get intersected areas - inp, res = mask.sindex.query_bulk(self.shapefile.geometry, predicate='intersects') + inp, res = shapefile.sindex.query_bulk(self.shapefile.geometry, predicate='intersects') print('Rank {0:03d}: {1} intersected areas found'.format(self.rank, len(inp))) # Calculate intersected areas and fractions - intersection = pd.concat([self.shapefile.geometry[inp].reset_index(), mask.geometry[res].reset_index()], + intersection = pd.concat([self.shapefile.geometry[inp].reset_index(), shapefile.geometry[res].reset_index()], axis=1, ignore_index=True) intersection.columns = (list(self.shapefile.geometry[inp].reset_index().columns) + - list(mask.geometry[res].reset_index().rename(columns={'geometry': 'geometry_mask', - 'index': 'index_mask'}).columns)) - intersection['area'] = intersection.apply(lambda x: x['geometry'].intersection(x['geometry_mask']).buffer(0).area, + list(shapefile.geometry[res].reset_index().rename(columns={'geometry': 'geometry_shapefile', + 'index': 'index_shapefile'}).columns)) + intersection['area'] = intersection.apply(lambda x: x['geometry'].intersection(x['geometry_shapefile']).buffer(0).area, axis=1) intersection['fraction'] = intersection.apply(lambda x: x['area'] / x['geometry'].area, axis=1) @@ -2634,26 +2644,30 @@ class Nes(object): intersection = intersection.drop_duplicates(subset='FID', keep="first") intersection = intersection.sort_values('FID').set_index('FID') - # Get data from mask - del mask['geometry'] - self.shapefile.loc[intersection.index, mask.columns] = np.array(mask.loc[intersection.index_mask, :]) + # Get data from shapefile + del shapefile['geometry'] + self.shapefile.loc[intersection.index, shapefile.columns] = np.array(shapefile.loc[intersection.index_shapefile, :]) - # Centroids that fall on the mask polygons, outside of the mask there will be NaN - elif method is None: + # Centroids that fall on the shapefile polygons, outside of the shapefile there will be NaN + elif method == 'centroid': # Make copy - shapefile_aux = deepcopy(self.shapefile) + aux_grid = deepcopy(self.shapefile) - # Get centroids of shapefile to mask - shapefile_aux.geometry = shapefile_aux.centroid + # Get centroids of shapefile to shapefile + aux_grid.geometry = aux_grid.centroid # Calculate spatial joint - shapefile_aux = gpd.sjoin(shapefile_aux, mask.to_crs(shapefile_aux.crs)) + aux_grid = gpd.sjoin(aux_grid, shapefile.to_crs(aux_grid.crs)) # Get data from shapes where there are centroids, rest will be NaN - del shapefile_aux['geometry'], shapefile_aux['index_right'] - self.shapefile.loc[shapefile_aux.index, shapefile_aux.columns] = shapefile_aux - + del aux_grid['geometry'], aux_grid['index_right'] + self.shapefile.loc[aux_grid.index, aux_grid.columns] = aux_grid + + else: + accepted_values = ['nearest', 'intersection', 'centroid'] + raise NotImplementedError('{0} is not implemented. Choose from: {1}'.format(method, accepted_values)) + return None def __gather_data_py_object(self): diff --git a/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb b/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb index d726d98..18cc82b 100644 --- a/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb +++ b/tutorials/1.Introduction/1.1.Read_Write_Regular.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# How to read and write regular grids" + "# How to read and write regular lat-lon grids" ] }, { @@ -24,7 +24,7 @@ "metadata": {}, "outputs": [], "source": [ - "nc_path_1 = '/gpfs/scratch/bsc32/bsc32538/original_files/franco_interp.nc'" + "nc_path_1 = '/gpfs/projects/bsc32/models/NES_tutorial_data/franco_interp.nc'" ] }, { @@ -414,7 +414,7 @@ " Domain: Regional\n", " Conventions: CF-1.7\n", " history: MONARCHv1.0 netcdf file.\n", - " comment: Generated on marenostrum4
    • sconcno2
      (time, lev, lat, lon)
      float32
      ...
      units :
      ppmV
      grid_mapping :
      crs
      [474375 values with dtype=float32]
    • crs
      ()
      |S1
      ...
      grid_mapping_name :
      latitude_longitude
      semi_major_axis :
      6371000.0
      inverse_flattening :
      0
      array(b'', dtype='|S1')
  • Domain :
    Regional
    Conventions :
    CF-1.7
    history :
    MONARCHv1.0 netcdf file.
    comment :
    Generated on marenostrum4
    • " ], "text/plain": [ "\n", @@ -520,7 +520,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -922,7 +922,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -1315,7 +1315,7 @@ " Domain: Regional\n", " Conventions: CF-1.7\n", " history: MONARCHv1.0 netcdf file.\n", - " comment: Generated on marenostrum4
    • sconcno2
      (time, lev, lat, lon)
      float32
      ...
      units :
      ppmV
      grid_mapping :
      crs
      [474375 values with dtype=float32]
    • crs
      ()
      |S1
      ...
      grid_mapping_name :
      latitude_longitude
      semi_major_axis :
      6371000.0
      inverse_flattening :
      0
      array(b'', dtype='|S1')
  • Domain :
    Regional
    Conventions :
    CF-1.7
    history :
    MONARCHv1.0 netcdf file.
    comment :
    Generated on marenostrum4
    • " ], "text/plain": [ "\n", diff --git a/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb b/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb index 290316f..abb2ad6 100644 --- a/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb +++ b/tutorials/1.Introduction/1.2.Read_Write_Rotated.ipynb @@ -23,7 +23,7 @@ "metadata": {}, "outputs": [], "source": [ - "nc_path_1 = '/gpfs/scratch/bsc32/bsc32538/mr_multiplyby/OUT/stats_bnds/monarch/a45g/regional/daily_max/O3_all/O3_all-000_2021080300.nc'" + "nc_path_1 = '/gpfs/projects/bsc32/models/NES_tutorial_data/O3_all-000_2021080300.nc'" ] }, { @@ -415,14 +415,14 @@ " rotated_pole |S1 b''\n", "Attributes:\n", " Conventions: CF-1.7\n", - " comment: Generated on marenostrum4
    • time_bnds
      (time, nv)
      datetime64[ns]
      ...
      array([['2021-08-03T00:00:00.000000000', '2021-08-07T00:00:00.000000000']],\n",
+       "      dtype='datetime64[ns]')
    • O3_all
      (time, lev, rlat, rlon)
      float32
      ...
      units :
      kg/m3
      long_name :
      TRACERS_044
      cell_methods :
      time: maximum (interval: 1hr)
      grid_mapping :
      rotated_pole
      [2282904 values with dtype=float32]
    • rotated_pole
      ()
      |S1
      ...
      grid_mapping_name :
      rotated_latitude_longitude
      grid_north_pole_latitude :
      39.0
      grid_north_pole_longitude :
      -170.0
      array(b'', dtype='|S1')
  • Conventions :
    CF-1.7
    comment :
    Generated on marenostrum4
    • " ], "text/plain": [ "\n", @@ -483,7 +483,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -810,7 +810,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -1205,14 +1205,14 @@ " rotated_pole |S1 b''\n", "Attributes:\n", " Conventions: CF-1.7\n", - " comment: Generated on marenostrum4
    • time_bnds
      (time, time_nv)
      datetime64[ns]
      ...
      array([['2021-08-03T00:00:00.000000000', '2021-08-07T00:00:00.000000000']],\n",
+       "      dtype='datetime64[ns]')
    • O3_all
      (time, lev, rlat, rlon)
      float32
      ...
      units :
      kg/m3
      long_name :
      TRACERS_044
      cell_methods :
      time: maximum (interval: 1hr)
      grid_mapping :
      rotated_pole
      [2282904 values with dtype=float32]
    • rotated_pole
      ()
      |S1
      ...
      grid_mapping_name :
      rotated_latitude_longitude
      grid_north_pole_latitude :
      39.0
      grid_north_pole_longitude :
      -170.0
      array(b'', dtype='|S1')
  • Conventions :
    CF-1.7
    comment :
    Generated on marenostrum4
    • " ], "text/plain": [ "\n", diff --git a/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb b/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb index 4f791b2..01e9f5b 100644 --- a/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb +++ b/tutorials/1.Introduction/1.3.Read_Write_Points.ipynb @@ -23,8 +23,7 @@ "metadata": {}, "outputs": [], "source": [ - "# nc_path_1 = '/esarchive/obs/eea/eionet/hourly/pm10/pm10_202107.nc' # EIONET\n", - "nc_path_1 = '/esarchive/obs/nilu/ebas/daily/pm10/pm10_201507.nc' # EBAS" + "nc_path_1 = '/gpfs/projects/bsc32/models/NES_tutorial_data/pm10_201507.nc' # EBAS" ] }, { @@ -418,7 +417,7 @@ " station_end_date (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n", " station_rural_back (station) |S75 b'nan' b'nan' ... b'nan' b'nan'\n", " latitude (station) float32 46.81 47.48 ... 53.33 38.88\n", - " station_ozone_classification (station) |S75 b'rural' b'rural' ... b'nan'
    • " ], "text/plain": [ "\n", @@ -710,7 +709,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -1408,7 +1407,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -1810,7 +1809,7 @@ " station_rural_back (station, strlen) object 'n' 'a' 'n' ... '' ''\n", " station_ozone_classification (station, strlen) object 'r' 'u' 'r' ... '' ''\n", "Attributes:\n", - " Conventions: CF-1.7
  • Conventions :
    CF-1.7
    • " ], "text/plain": [ "\n", @@ -2415,7 +2414,7 @@ " source: Surface observations\n", " creator_name: Dene R. Bowdalo\n", " creator_email: dene.bowdalo@bsc.es\n", - " version: 1.4
  • title :
    Surface sulphate data in the EANET network in 2019-11.
    institution :
    Barcelona Supercomputing Center
    source :
    Surface observations
    creator_name :
    Dene R. Bowdalo
    creator_email :
    dene.bowdalo@bsc.es
    version :
    1.4
    • " ], "text/plain": [ "\n", @@ -2641,7 +2640,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -3341,14 +3340,17 @@ "Rank 000: Var Joly-Peuch_classification_code data (40/173)\n", "Rank 000: Var Joly-Peuch_classification_code completed (40/173)\n", "Rank 000: Writing Koppen-Geiger_classification var (41/173)\n", - "Rank 000: Var Koppen-Geiger_classification created (41/173)\n" + "Rank 000: Var Koppen-Geiger_classification created (41/173)\n", + "Rank 000: Var Koppen-Geiger_classification data (41/173)\n", + "Rank 000: Var Koppen-Geiger_classification completed (41/173)\n", + "Rank 000: Writing Koppen-Geiger_modal_classification_25km var (42/173)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes_ghost.py:592: UserWarning: WARNING!!! GHOST datasets cannot be written in parallel yet. Changing to serial mode.\n", + "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes_ghost.py:593: UserWarning: WARNING!!! GHOST datasets cannot be written in parallel yet. Changing to serial mode.\n", " warnings.warn(msg)\n" ] }, @@ -3356,9 +3358,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "Rank 000: Var Koppen-Geiger_classification data (41/173)\n", - "Rank 000: Var Koppen-Geiger_classification completed (41/173)\n", - "Rank 000: Writing Koppen-Geiger_modal_classification_25km var (42/173)\n", "Rank 000: Var Koppen-Geiger_modal_classification_25km created (42/173)\n", "Rank 000: Var Koppen-Geiger_modal_classification_25km data (42/173)\n", "Rank 000: Var Koppen-Geiger_modal_classification_25km completed (42/173)\n", @@ -4262,7 +4261,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 26, @@ -4669,7 +4668,7 @@ " creator_name: Dene R. Bowdalo\n", " creator_email: dene.bowdalo@bsc.es\n", " version: 1.4\n", - " Conventions: CF-1.7