From 53e79f185b08d74fb5a1a0a0e3c6f6bc4477bbb6 Mon Sep 17 00:00:00 2001 From: Alba Vilanova Cortezon Date: Mon, 17 Oct 2022 12:56:42 +0200 Subject: [PATCH] Add CSIC example --- .../2.4.Create_Points_Port_Barcelona.ipynb | 791 +++----------- .../2.Creation/2.5.Create_Points_CSIC.ipynb | 968 ++++++++++++++++++ ....Create-LCC.ipynb => 2.6.Create-LCC.ipynb} | 0 ...rcator.ipynb => 2.7.Create_Mercator.ipynb} | 0 4 files changed, 1117 insertions(+), 642 deletions(-) create mode 100644 tutorials/2.Creation/2.5.Create_Points_CSIC.ipynb rename tutorials/2.Creation/{2.5.Create-LCC.ipynb => 2.6.Create-LCC.ipynb} (100%) rename tutorials/2.Creation/{2.6.Create_Mercator.ipynb => 2.7.Create_Mercator.ipynb} (100%) diff --git a/tutorials/2.Creation/2.4.Create_Points_Port_Barcelona.ipynb b/tutorials/2.Creation/2.4.Create_Points_Port_Barcelona.ipynb index e66b4f8..afa457c 100644 --- a/tutorials/2.Creation/2.4.Create_Points_Port_Barcelona.ipynb +++ b/tutorials/2.Creation/2.4.Create_Points_Port_Barcelona.ipynb @@ -166,7 +166,7 @@ } ], "source": [ - "file_path = '/esarchive/scratch/avilanova/software/NES/Jupyter_notebooks/input/Dades_Port_Barcelona_2017-2021_corr.xlsx'\n", + "file_path = '/esarchive/scratch/avilanova/software/NES/tutorials/data/Dades_Port_Barcelona_2017-2021_corr.xlsx'\n", "df_data = pd.read_excel(file_path, header=3, index_col='Horario: UTC')\n", "df_data" ] @@ -279,7 +279,7 @@ } ], "source": [ - "file_path = '/esarchive/scratch/avilanova/software/NES/Jupyter_notebooks/input/estaciones.csv'\n", + "file_path = '/esarchive/scratch/avilanova/software/NES/tutorials/data/estaciones.csv'\n", "df_stations = pd.read_csv(file_path)\n", "df_stations" ] @@ -404,10 +404,14 @@ "Rank 000: Creating points_grid_no2.nc\n", "Rank 000: NetCDF ready to write\n", "Rank 000: Dimensions done\n", + "|S1\n", + "S75\n", "Rank 000: Writing station_name var (1/2)\n", "Rank 000: Var station_name created (1/2)\n", "Rank 000: Var station_name data (1/2)\n", "Rank 000: Var station_name completed (1/2)\n", + "float64\n", + "\n", "Rank 000: Writing sconcno2 var (2/2)\n", "Rank 000: Var sconcno2 created (2/2)\n" ] @@ -416,7 +420,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes.py:405: UserWarning: WARNING!!! Different data types for variable station_nameInput dtype=, data dtype=object\n", + "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes.py:338: UserWarning: WARNING!!! Different data types for variable station_nameInput dtype=, data dtype=object\n", " warnings.warn(msg)\n" ] }, @@ -794,38 +798,49 @@ " fill: currentColor;\n", "}\n", "
<xarray.Dataset>\n",
-       "Dimensions:       (time: 43818, station: 2)\n",
+       "Dimensions:       (time: 43818, station: 2, strlen: 75)\n",
        "Coordinates:\n",
        "  * time          (time) datetime64[ns] 2017-01-01 ... 2021-12-31T17:00:00\n",
        "  * station       (station) float64 0.0 1.0\n",
+       "Dimensions without coordinates: strlen\n",
        "Data variables:\n",
-       "    station_name  (station) object 'NO2-UM' 'NO2-ZAL Prat'\n",
-       "    sconcno2      (time, station) float64 64.64 49.08 68.16 ... 12.76 28.66\n",
        "    lat           (station) float64 41.37 41.32\n",
        "    lon           (station) float64 2.185 2.135\n",
+       "    station_name  (station, strlen) object 'N' 'O' '2' '-' 'U' ... '' '' '' ''\n",
+       "    sconcno2      (time, station) float64 64.64 49.08 68.16 ... 12.76 28.66\n",
        "Attributes:\n",
-       "    Conventions:  CF-1.7
  • Conventions :
    CF-1.7
    • " ], "text/plain": [ "\n", - "Dimensions: (time: 43818, station: 2)\n", + "Dimensions: (time: 43818, station: 2, strlen: 75)\n", "Coordinates:\n", " * time (time) datetime64[ns] 2017-01-01 ... 2021-12-31T17:00:00\n", " * station (station) float64 0.0 1.0\n", + "Dimensions without coordinates: strlen\n", "Data variables:\n", - " station_name (station) object ...\n", - " sconcno2 (time, station) float64 ...\n", " lat (station) float64 ...\n", " lon (station) float64 ...\n", + " station_name (station, strlen) object ...\n", + " sconcno2 (time, station) float64 ...\n", "Attributes:\n", " Conventions: CF-1.7" ] @@ -840,259 +855,138 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 13, "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/esarchive/scratch/avilanova/software/NES/nes/load_nes.py:69: UserWarning: Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\n", + " warnings.warn(\"Parallel method cannot be 'Y' to create points NES. Setting it to 'X'\")\n" + ] + } + ], "source": [ - "## Create one dataset per month (Ready for Providentia)" + "a = open_netcdf('points_grid_no2.nc')" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[['N' 'O' '2' '-' 'U' 'M' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' ''\n", + " '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' ''\n", + " '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' ''\n", + " '' '' '' '' '']\n", + " ['N' 'O' '2' '-' 'Z' 'A' 'L' ' ' 'P' 'r' 'a' 't' '' '' '' '' '' '' '' ''\n", + " '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' ''\n", + " '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' ''\n", + " '' '' '' '' '' '' '']] (2, 75) object\n" + ] + } + ], "source": [ - "df_data['month'] = df_data.index.month\n", - "df_data['year'] = df_data.index.year" + "a.load()" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    SO2-DarsenaSO2-UMNO2-UMNO2-ZAL Pratmonthyear
    2017-01-01 00:00:000.20NaN64.6449.0812017
    2017-01-01 01:00:000.18NaN68.1653.0012017
    2017-01-01 02:00:000.04NaN68.2946.7512017
    2017-01-01 03:00:000.00NaN62.9139.6512017
    2017-01-01 04:00:000.00NaN47.1328.8612017
    .....................
    2021-12-31 13:00:000.77NaN35.3824.96122021
    2021-12-31 14:00:000.37NaN35.8024.16122021
    2021-12-31 15:00:000.23NaN29.1025.79122021
    2021-12-31 16:00:000.14NaN9.2429.82122021
    2021-12-31 17:00:000.20NaN12.7628.66122021
    \n", - "

    43818 rows × 6 columns

    \n", - "
    " - ], "text/plain": [ - " SO2-Darsena SO2-UM NO2-UM NO2-ZAL Prat month year\n", - "2017-01-01 00:00:00 0.20 NaN 64.64 49.08 1 2017\n", - "2017-01-01 01:00:00 0.18 NaN 68.16 53.00 1 2017\n", - "2017-01-01 02:00:00 0.04 NaN 68.29 46.75 1 2017\n", - "2017-01-01 03:00:00 0.00 NaN 62.91 39.65 1 2017\n", - "2017-01-01 04:00:00 0.00 NaN 47.13 28.86 1 2017\n", - "... ... ... ... ... ... ...\n", - "2021-12-31 13:00:00 0.77 NaN 35.38 24.96 12 2021\n", - "2021-12-31 14:00:00 0.37 NaN 35.80 24.16 12 2021\n", - "2021-12-31 15:00:00 0.23 NaN 29.10 25.79 12 2021\n", - "2021-12-31 16:00:00 0.14 NaN 9.24 29.82 12 2021\n", - "2021-12-31 17:00:00 0.20 NaN 12.76 28.66 12 2021\n", - "\n", - "[43818 rows x 6 columns]" + "{'station': {'data': masked_array(data=[0., 1.],\n", + " mask=False,\n", + " fill_value=1e+20),\n", + " 'dimensions': ('station',),\n", + " 'units': '',\n", + " 'axis': 'X',\n", + " 'long_name': '',\n", + " 'standard_name': 'station'},\n", + " 'station_name': {'data': array(['NO2-UM', 'NO2-ZAL Prat'], dtype='\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'stop' is not defined" ] } ], + "source": [ + "stop" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create one dataset per month (Ready for Providentia)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data['month'] = df_data.index.month\n", + "df_data['year'] = df_data.index.year" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "for (year, month), current in df_data.groupby(['year', 'month']):\n", "\n", @@ -1122,9 +1016,15 @@ " \n", " # Assign metadata\n", " points_grid.variables = metadata\n", - " \n", + " \n", + "\n", + " # Making directory\n", + " netcdf_path = '/esarchive/obs/port_barcelona/port-barcelona/hourly/sconcno2'\n", + " if not os.path.exists(os.path.dirname(netcdf_path)):\n", + " os.makedirs(os.path.dirname(netcdf_path))\n", + " \n", " # Save files\n", - " points_grid.to_netcdf('/esarchive/obs/generalitat/port-barcelona/hourly/sconcno2/sconcno2_{0}{1}.nc'.format(year, str(month).zfill(2)))\n", + " points_grid.to_netcdf(netcdf_path + '/sconcno2_{0}{1}.nc'.format(year, str(month).zfill(2)))\n", " \n", " del points_grid\n", " print('Done sconcno2_{0}{1}.nc'.format(year, str(month).zfill(2)))" @@ -1132,7 +1032,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -1495,22 +1395,22 @@ " * time (time) datetime64[ns] 2017-05-01 ... 2017-05-31T23:00:00\n", " * station (station) float64 0.0 1.0\n", "Data variables:\n", - " station_name (station) object 'NO2-UM' 'NO2-ZAL Prat'\n", + " station_name (station) |S75 b'NO2-UM' b'NO2-ZAL Prat'\n", " altitude (station) float64 nan nan\n", " sconcno2 (time, station) float64 8.77 23.49 1.76 ... 27.11 13.87 30.07\n", " lat (station) float64 41.37 41.32\n", " lon (station) float64 2.185 2.135\n", "Attributes:\n", - " Conventions: CF-1.7
  • Conventions :
    CF-1.7
    • " ], "text/plain": [ "\n", @@ -1519,7 +1419,7 @@ " * time (time) datetime64[ns] 2017-05-01 ... 2017-05-31T23:00:00\n", " * station (station) float64 0.0 1.0\n", "Data variables:\n", - " station_name (station) object ...\n", + " station_name (station) |S75 ...\n", " altitude (station) float64 ...\n", " sconcno2 (time, station) float64 ...\n", " lat (station) float64 ...\n", @@ -1528,420 +1428,22 @@ " Conventions: CF-1.7" ] }, - "execution_count": 23, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "xr.open_dataset('/esarchive/obs/generalitat/port-barcelona/hourly/sconcno2/sconcno2_201705.nc')" + "xr.open_dataset('/esarchive/obs/port_barcelona/port-barcelona/hourly/sconcno2/sconcno2_201705.nc')" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset>\n",
-       "Dimensions:       (time: 738, station: 2)\n",
-       "Coordinates:\n",
-       "  * time          (time) datetime64[ns] 2021-12-01 ... 2021-12-31T17:00:00\n",
-       "  * station       (station) float64 0.0 1.0\n",
-       "Data variables:\n",
-       "    station_name  (station) object 'NO2-UM' 'NO2-ZAL Prat'\n",
-       "    altitude      (station) float64 nan nan\n",
-       "    sconcno2      (time, station) float64 13.33 nan 13.42 ... 29.82 12.76 28.66\n",
-       "    lat           (station) float64 41.37 41.32\n",
-       "    lon           (station) float64 2.185 2.135\n",
-       "Attributes:\n",
-       "    Conventions:  CF-1.7
    " - ], - "text/plain": [ - "\n", - "Dimensions: (time: 738, station: 2)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2021-12-01 ... 2021-12-31T17:00:00\n", - " * station (station) float64 0.0 1.0\n", - "Data variables:\n", - " station_name (station) object ...\n", - " altitude (station) float64 ...\n", - " sconcno2 (time, station) float64 ...\n", - " lat (station) float64 ...\n", - " lon (station) float64 ...\n", - "Attributes:\n", - " Conventions: CF-1.7" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "xr.open_dataset('/esarchive/obs/generalitat/port-barcelona/hourly/sconcno2/sconcno2_202112.nc')" + "xr.open_dataset('/esarchive/obs/port_barcelona/port-barcelona/hourly/sconcno2/sconcno2_202112.nc')" ] } ], @@ -1962,6 +1464,11 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } } }, "nbformat": 4, diff --git a/tutorials/2.Creation/2.5.Create_Points_CSIC.ipynb b/tutorials/2.Creation/2.5.Create_Points_CSIC.ipynb new file mode 100644 index 0000000..fd02dd3 --- /dev/null +++ b/tutorials/2.Creation/2.5.Create_Points_CSIC.ipynb @@ -0,0 +1,968 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to create monthly observations datasets (CSIC)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "import pandas as pd\n", + "import numpy as np\n", + "from datetime import datetime, timedelta\n", + "from nes import *\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    traffic_siteurban_site
    Date-hour in
    2019-01-014.9889882.553235
    2019-02-013.4225351.556226
    2019-03-012.6750651.686355
    2019-04-013.4255221.975486
    2019-05-015.3148091.119245
    2019-06-013.1394951.626567
    2019-07-010.0000002.226856
    2019-08-010.0000002.469638
    2019-09-010.0000003.727355
    2019-10-010.0000001.535056
    2019-11-010.0000002.511152
    2019-12-010.0000000.000000
    \n", + "
    " + ], + "text/plain": [ + " traffic_site urban_site\n", + "Date-hour in \n", + "2019-01-01 4.988988 2.553235\n", + "2019-02-01 3.422535 1.556226\n", + "2019-03-01 2.675065 1.686355\n", + "2019-04-01 3.425522 1.975486\n", + "2019-05-01 5.314809 1.119245\n", + "2019-06-01 3.139495 1.626567\n", + "2019-07-01 0.000000 2.226856\n", + "2019-08-01 0.000000 2.469638\n", + "2019-09-01 0.000000 3.727355\n", + "2019-10-01 0.000000 1.535056\n", + "2019-11-01 0.000000 2.511152\n", + "2019-12-01 0.000000 0.000000" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "file_path = '/esarchive/scratch/avilanova/software/NES/tutorials/data/NH3_barcelona_2019_csic.csv'\n", + "df_data = pd.read_csv(file_path, index_col='Date-hour in', parse_dates=True)\n", + "df_data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    stationLonLat
    0urban_site2.115141.3875
    1traffic_site2.153441.3987
    \n", + "
    " + ], + "text/plain": [ + " station Lon Lat\n", + "0 urban_site 2.1151 41.3875\n", + "1 traffic_site 2.1534 41.3987" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "file_path = '/esarchive/scratch/avilanova/software/NES/tutorials/data/NH3_stations_CSIC.csv'\n", + "df_stations = pd.read_csv(file_path)\n", + "df_stations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create dataset with all timesteps" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([datetime.datetime(2019, 1, 1, 0, 0),\n", + " datetime.datetime(2019, 2, 1, 0, 0),\n", + " datetime.datetime(2019, 3, 1, 0, 0),\n", + " datetime.datetime(2019, 4, 1, 0, 0),\n", + " datetime.datetime(2019, 5, 1, 0, 0),\n", + " datetime.datetime(2019, 6, 1, 0, 0),\n", + " datetime.datetime(2019, 7, 1, 0, 0),\n", + " datetime.datetime(2019, 8, 1, 0, 0),\n", + " datetime.datetime(2019, 9, 1, 0, 0),\n", + " datetime.datetime(2019, 10, 1, 0, 0),\n", + " datetime.datetime(2019, 11, 1, 0, 0),\n", + " datetime.datetime(2019, 12, 1, 0, 0)], dtype=object)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "times = df_data.index.to_pydatetime()\n", + "times" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([41.3875, 41.3987])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lat = df_stations['Lat'].to_numpy()\n", + "lat" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([2.1151, 2.1534])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lon = df_stations['Lon'].to_numpy()\n", + "lon" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[4.98898815, 2.55323513],\n", + " [3.42253493, 1.55622608],\n", + " [2.67506499, 1.68635512],\n", + " [3.42552223, 1.97548632],\n", + " [5.31480909, 1.11924537],\n", + " [3.139495 , 1.6265667 ],\n", + " [0. , 2.22685565],\n", + " [0. , 2.46963801],\n", + " [0. , 3.72735536],\n", + " [0. , 1.5350561 ],\n", + " [0. , 2.51115189],\n", + " [0. , 0. ]])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_data.to_numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "metadata = {'station_name': {'data': df_data.columns.to_numpy(),\n", + " 'dimensions': ('station',),\n", + " 'dtype': str},\n", + " 'sconcnh3': {'data': df_data.to_numpy(),\n", + " 'dimensions': ('time', 'station',),\n", + " 'dtype': float}}" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "points_grid = create_nes(comm=None, info=False, projection=None, parallel_method='X',\n", + " lat=lat, lon=lon, times=times)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "points_grid.variables = metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank 000: Creating points_grid_nh3.nc\n", + "Rank 000: NetCDF ready to write\n", + "Rank 000: Dimensions done\n", + "Rank 000: Writing station_name var (1/2)\n", + "Rank 000: Var station_name created (1/2)\n", + "Rank 000: Var station_name data (1/2)\n", + "Rank 000: Var station_name completed (1/2)\n", + "Rank 000: Writing sconcnh3 var (2/2)\n", + "Rank 000: Var sconcnh3 created (2/2)\n", + "Rank 000: Var sconcnh3 data (2/2)\n", + "Rank 000: Var sconcnh3 completed (2/2)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/esarchive/scratch/avilanova/software/NES/nes/nc_projections/points_nes.py:336: UserWarning: WARNING!!! Different data types for variable station_nameInput dtype=, data dtype=object\n", + " warnings.warn(msg)\n" + ] + } + ], + "source": [ + "points_grid.to_netcdf('points_grid_nh3.nc', info=True)\n", + "del points_grid" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset>\n",
+       "Dimensions:       (time: 12, station: 2, strlen: 75)\n",
+       "Coordinates:\n",
+       "  * time          (time) datetime64[ns] 2019-01-01 2019-02-01 ... 2019-12-01\n",
+       "  * station       (station) float64 0.0 1.0\n",
+       "Dimensions without coordinates: strlen\n",
+       "Data variables:\n",
+       "    lat           (station) float64 41.39 41.4\n",
+       "    lon           (station) float64 2.115 2.153\n",
+       "    station_name  (station, strlen) object 't' 'r' 'a' 'f' 'f' ... '' '' '' ''\n",
+       "    sconcnh3      (time, station) float64 4.989 2.553 3.423 ... 2.511 0.0 0.0\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.7
    " + ], + "text/plain": [ + "\n", + "Dimensions: (time: 12, station: 2, strlen: 75)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2019-01-01 2019-02-01 ... 2019-12-01\n", + " * station (station) float64 0.0 1.0\n", + "Dimensions without coordinates: strlen\n", + "Data variables:\n", + " lat (station) float64 ...\n", + " lon (station) float64 ...\n", + " station_name (station, strlen) object ...\n", + " sconcnh3 (time, station) float64 ...\n", + "Attributes:\n", + " Conventions: CF-1.7" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xr.open_dataset('points_grid_nh3.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'stop' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'stop' is not defined" + ] + } + ], + "source": [ + "stop" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create one dataset per month (Ready for Providentia)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data['month'] = df_data.index.month\n", + "df_data['year'] = df_data.index.year" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for (year, month), current in df_data.groupby(['year', 'month']):\n", + " \n", + " # Read time\n", + " times = current.index.to_pydatetime()\n", + " \n", + " # Fill altitude with nans\n", + " altitude = np.full(len(current.columns[0:2]), np.nan)\n", + " \n", + " # Read metadata\n", + " metadata = {'station_name': {'data': current.columns[0:2].to_numpy(),\n", + " 'dimensions': ('station'),\n", + " 'standard_name': ''},\n", + " 'altitude': {'data': altitude,\n", + " 'dimensions': ('station',),\n", + " 'units': 'meters',\n", + " 'standard_name': 'altitude'},\n", + " 'sconcnh3': {'data': current.iloc[:, 0:2].to_numpy(),\n", + " 'units': 'µg m-3',\n", + " 'dimensions': ('time', 'station',),\n", + " 'long_name': ''}\n", + " }\n", + " \n", + " # Create object\n", + " points_grid = create_nes(comm=None, info=False, projection=None, parallel_method='X',\n", + " lat=lat, lon=lon, times=times)\n", + " \n", + " # Assign metadata\n", + " points_grid.variables = metadata\n", + " \n", + " # Making directory\n", + " netcdf_path = '/esarchive/obs/csic/csic/monthly/sconcnh3/'\n", + " if not os.path.exists(os.path.dirname(netcdf_path)):\n", + " os.makedirs(os.path.dirname(netcdf_path))\n", + " \n", + " # Save files\n", + " points_grid.to_netcdf(netcdf_path + '/sconcnh3_{0}{1}.nc'.format(year, str(month).zfill(2)))\n", + " \n", + " del points_grid\n", + " print('Done sconcnh3_{0}{1}.nc'.format(year, str(month).zfill(2)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xr.open_dataset('/esarchive/obs/csic/csic/monthly/sconcnh3/sconcnh3_201905.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xr.open_dataset('/esarchive/obs/csic/csic/monthly/sconcnh3/sconcnh3_201901.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tutorials/2.Creation/2.5.Create-LCC.ipynb b/tutorials/2.Creation/2.6.Create-LCC.ipynb similarity index 100% rename from tutorials/2.Creation/2.5.Create-LCC.ipynb rename to tutorials/2.Creation/2.6.Create-LCC.ipynb diff --git a/tutorials/2.Creation/2.6.Create_Mercator.ipynb b/tutorials/2.Creation/2.7.Create_Mercator.ipynb similarity index 100% rename from tutorials/2.Creation/2.6.Create_Mercator.ipynb rename to tutorials/2.Creation/2.7.Create_Mercator.ipynb -- GitLab