{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How to create monthly observations datasets (CSIC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nes\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime, timedelta\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Collect data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have two dataframes: the first one gives us NH3 monthly data at some stations, and the second one gives us the locations of the stations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>traffic_site</th>\n",
       "      <th>urban_site</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date-hour in</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-01</th>\n",
       "      <td>4.988988</td>\n",
       "      <td>2.553235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-01</th>\n",
       "      <td>3.422535</td>\n",
       "      <td>1.556226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-01</th>\n",
       "      <td>2.675065</td>\n",
       "      <td>1.686355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-01</th>\n",
       "      <td>3.425522</td>\n",
       "      <td>1.975486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-01</th>\n",
       "      <td>5.314809</td>\n",
       "      <td>1.119245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-01</th>\n",
       "      <td>3.139495</td>\n",
       "      <td>1.626567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-07-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.226856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.469638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-09-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.727355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.535056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-11-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.511152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-12-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              traffic_site  urban_site\n",
       "Date-hour in                          \n",
       "2019-01-01        4.988988    2.553235\n",
       "2019-02-01        3.422535    1.556226\n",
       "2019-03-01        2.675065    1.686355\n",
       "2019-04-01        3.425522    1.975486\n",
       "2019-05-01        5.314809    1.119245\n",
       "2019-06-01        3.139495    1.626567\n",
       "2019-07-01        0.000000    2.226856\n",
       "2019-08-01        0.000000    2.469638\n",
       "2019-09-01        0.000000    3.727355\n",
       "2019-10-01        0.000000    1.535056\n",
       "2019-11-01        0.000000    2.511152\n",
       "2019-12-01        0.000000    0.000000"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = '/gpfs/projects/bsc32/models/NES_tutorial_data/NH3_barcelona_2019_csic.csv'\n",
    "df_nh3 = pd.read_csv(file_path, index_col='Date-hour in', parse_dates=True)\n",
    "df_nh3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station</th>\n",
       "      <th>Lon</th>\n",
       "      <th>Lat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>urban_site</td>\n",
       "      <td>2.1151</td>\n",
       "      <td>41.3875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>traffic_site</td>\n",
       "      <td>2.1534</td>\n",
       "      <td>41.3987</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        station     Lon      Lat\n",
       "0    urban_site  2.1151  41.3875\n",
       "1  traffic_site  2.1534  41.3987"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = '/gpfs/projects/bsc32/models/NES_tutorial_data/NH3_stations_CSIC.csv'\n",
    "df_stations = pd.read_csv(file_path)\n",
    "df_stations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Create dataset with all timesteps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "times = df_nh3.index.to_pydatetime()\n",
    "lat = df_stations['Lat'].to_numpy()\n",
    "lon = df_stations['Lon'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nessy = nes.create_nes(comm=None, info=False, projection=None, parallel_method='X',\n",
    "                   lat=lat, lon=lon, times=times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "variables = {'station_name': {'data': df_nh3.columns.to_numpy(),\n",
    "                              'dimensions': ('station',),\n",
    "                              'dtype': str},\n",
    "             'sconcnh3': {'data': df_nh3.to_numpy(),\n",
    "                          'dimensions': ('time', 'station',),\n",
    "                          'dtype': float}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nessy.variables = variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rank 000: Creating points_csic_nh3.nc\n",
      "Rank 000: NetCDF ready to write\n",
      "Rank 000: Dimensions done\n",
      "Rank 000: Writing station_name var (1/2)\n",
      "Rank 000: Var station_name created (1/2)\n",
      "Rank 000: Filling station_name)\n",
      "Rank 000: Var station_name data (1/2)\n",
      "Rank 000: Var station_name completed (1/2)\n",
      "Rank 000: Writing sconcnh3 var (2/2)\n",
      "Rank 000: Var sconcnh3 created (2/2)\n",
      "Rank 000: Filling sconcnh3)\n",
      "Rank 000: Var sconcnh3 data (2/2)\n",
      "Rank 000: Var sconcnh3 completed (2/2)\n"
     ]
    }
   ],
   "source": [
    "nessy.set_strlen(75)\n",
    "nessy.to_netcdf('points_csic_nh3.nc', info=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Create one dataset per month (Ready for Providentia)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add columns with month and year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>traffic_site</th>\n",
       "      <th>urban_site</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date-hour in</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-01-01</th>\n",
       "      <td>4.988988</td>\n",
       "      <td>2.553235</td>\n",
       "      <td>1</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-02-01</th>\n",
       "      <td>3.422535</td>\n",
       "      <td>1.556226</td>\n",
       "      <td>2</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-03-01</th>\n",
       "      <td>2.675065</td>\n",
       "      <td>1.686355</td>\n",
       "      <td>3</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-01</th>\n",
       "      <td>3.425522</td>\n",
       "      <td>1.975486</td>\n",
       "      <td>4</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-05-01</th>\n",
       "      <td>5.314809</td>\n",
       "      <td>1.119245</td>\n",
       "      <td>5</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-06-01</th>\n",
       "      <td>3.139495</td>\n",
       "      <td>1.626567</td>\n",
       "      <td>6</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-07-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.226856</td>\n",
       "      <td>7</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-08-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.469638</td>\n",
       "      <td>8</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-09-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.727355</td>\n",
       "      <td>9</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-10-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.535056</td>\n",
       "      <td>10</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-11-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.511152</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-12-01</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              traffic_site  urban_site  month  year\n",
       "Date-hour in                                       \n",
       "2019-01-01        4.988988    2.553235      1  2019\n",
       "2019-02-01        3.422535    1.556226      2  2019\n",
       "2019-03-01        2.675065    1.686355      3  2019\n",
       "2019-04-01        3.425522    1.975486      4  2019\n",
       "2019-05-01        5.314809    1.119245      5  2019\n",
       "2019-06-01        3.139495    1.626567      6  2019\n",
       "2019-07-01        0.000000    2.226856      7  2019\n",
       "2019-08-01        0.000000    2.469638      8  2019\n",
       "2019-09-01        0.000000    3.727355      9  2019\n",
       "2019-10-01        0.000000    1.535056     10  2019\n",
       "2019-11-01        0.000000    2.511152     11  2019\n",
       "2019-12-01        0.000000    0.000000     12  2019"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nh3['month'] = df_nh3.index.month\n",
    "df_nh3['year'] = df_nh3.index.year\n",
    "df_nh3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Iterate through each month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201901.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n",
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201902.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201903.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201904.nc\n",
      "Done sconcnh3_201905.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201906.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n",
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201907.nc\n",
      "Done sconcnh3_201908.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201909.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n",
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201910.nc\n",
      "Done sconcnh3_201911.nc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bsc/bsc032781/.local/lib/python3.9/site-packages/nes/nc_projections/points_nes.py:368: UserWarning: WARNING!!! Different data types for variable station_name. Input dtype=<class 'str'>. Data dtype=object.\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done sconcnh3_201912.nc\n"
     ]
    }
   ],
   "source": [
    "for (year, month), current in df_nh3.groupby(['year', 'month']):\n",
    "    \n",
    "    # Read time\n",
    "    times = current.index.to_pydatetime()\n",
    "    \n",
    "    # Fill altitude with nans\n",
    "    altitude = np.full(len(current.columns[0:2]), np.nan)\n",
    "  \n",
    "    # Read metadata\n",
    "    metadata = {'station_name': {'data': current.columns[0:2].to_numpy(),\n",
    "                                 'dimensions': ('station',),\n",
    "                                 'dtype': str,\n",
    "                                 'standard_name': ''},\n",
    "                'altitude': {'data': altitude,\n",
    "                             'dimensions': ('station',),\n",
    "                             'units': 'meters',\n",
    "                             'standard_name': 'altitude'},\n",
    "                'sconcnh3': {'data': current.iloc[:, 0:2].to_numpy(),\n",
    "                             'units': 'µg m-3',\n",
    "                             'dimensions': ('time', 'station',),\n",
    "                             'long_name': ''}\n",
    "               }\n",
    "    \n",
    "    # Create object\n",
    "    points_grid = nes.create_nes(comm=None, info=False, projection=None, parallel_method='X',\n",
    "                                 lat=lat, lon=lon, times=times)\n",
    "    \n",
    "    # Assign metadata\n",
    "    points_grid.variables = metadata\n",
    "    points_grid.set_strlen(75)\n",
    "    \n",
    "    # Making directory\n",
    "    netcdf_path = 'csic/csic/monthly/sconcnh3/'\n",
    "    if not os.path.exists(os.path.dirname(netcdf_path)):\n",
    "        os.makedirs(os.path.dirname(netcdf_path))\n",
    "        \n",
    "    # To run Providentia, this folder should be moved to:\n",
    "    # '/esarchive/obs/' as in '/esarchive/obs/csic/csic/monthly/sconcnh3/'\n",
    "    \n",
    "    # Save files\n",
    "    points_grid.to_netcdf(netcdf_path + '/sconcnh3_{0}{1}.nc'.format(year, str(month).zfill(2)))\n",
    "    \n",
    "    del points_grid\n",
    "    print('Done sconcnh3_{0}{1}.nc'.format(year, str(month).zfill(2)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
