{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7343975a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time [0.]\n",
      "np.shape(series) (100, 1, 1)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 58356 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "class TimeSeries:\n",
    "    \n",
    "\n",
    "    def generate_time_series(self,batch_size, n_steps):\n",
    "        freq1, freq2, offset1, offset2 = np.random.rand(4, batch_size, 1)\n",
    "        time = np.linspace(0,1, n_steps)\n",
    "        print(\"time\",time)\n",
    "        series = 0.5*np.sin((time - offset1))*(freq1*10+10) # Welle 1\n",
    "        series += 0.2*np.sin((time - offset2))*(freq2*20+20) # Welle 2\n",
    "        series += 0.01*(np.random.rand(batch_size, n_steps)-0.5) # Rauschen\n",
    "        return series[..., np.newaxis].astype(np.float32)\n",
    "\n",
    "timeseries = TimeSeries()\n",
    "\n",
    "# Generate timeseries with 50 data points and 7 time steps stapled over each other\n",
    "#series = timeseries.generate_time_series(50, 7)\n",
    "data = 100 # Number of data points\n",
    "n_steps = 1 # Number of stacked curves\n",
    "series = timeseries.generate_time_series(data, n_steps)\n",
    "print(\"np.shape(series)\",np.shape(series))\n",
    "#print(series[:])\n",
    "# Create x axis of 50 data points within (-10,10) range\n",
    "#x = np.linspace(-10,10,50)\n",
    "x = np.linspace(-10,10,data)\n",
    "fig = plt.figure()\n",
    "# Plot x vs the series of 4 stapled time series \n",
    "#plt.plot(x, series[0:50,0:4,0], '-')\n",
    "plt.plot(x, series[0:,:,0], '-')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "48ef1815",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "281975a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data = train_test_split(series, test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "03b04ab3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80 20\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data),len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "da16111a",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = np.linspace(0,50,data)\n",
    "x_train_data, x_test_data = train_test_split(t, test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "00af0cc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80 20\n"
     ]
    }
   ],
   "source": [
    "print(len(x_train_data),len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a84c35bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.50505051 1.01010101 1.51515152 2.02020202 2.52525253\n",
      " 3.03030303 3.53535354 4.04040404 4.54545455] [40.4040404  40.90909091 41.41414141 41.91919192 42.42424242 42.92929293\n",
      " 43.43434343 43.93939394 44.44444444 44.94949495]\n"
     ]
    }
   ],
   "source": [
    "print(x_train_data[:10],x_test_data[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a7ca3bd",
   "metadata": {},
   "source": [
    " # Linear Regression\n",
    " **Simple linear regression function to check for performance**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6dc8c6e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80,) (80,)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LinearRegression()\n",
    "print(np.shape(x_train_data),np.shape(train_data[:,0,0]))\n",
    "model.fit(x_train_data.reshape(-1, 1), train_data[:,0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0660f8f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method LinearRegression.fit of LinearRegression()>\n"
     ]
    }
   ],
   "source": [
    "print(model.fit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "439d091f",
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_values = model.predict(x_test_data.reshape(-1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "08dda57e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicted_values [-6.4764651  -6.47928248 -6.48209986 -6.48491724 -6.48773462 -6.49055199\n",
      " -6.49336937 -6.49618675 -6.49900413 -6.50182151 -6.50463889 -6.50745627\n",
      " -6.51027365 -6.51309103 -6.51590841 -6.51872579 -6.52154316 -6.52436054\n",
      " -6.52717792 -6.5299953 ]\n"
     ]
    }
   ],
   "source": [
    "print(\"predicted_values\",predicted_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "74dca538",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE: 6.458035406766351\n",
      "MAE: 2.021223135075824\n",
      "RMSE: 2.5412664966048624\n",
      "R2 Score: -0.0009080971778105962\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "y_test = test_data[:,0,0]\n",
    "#actual_values = y_test.values.reshape(-1,1)\n",
    "# Assuming you have the actual values in a variable named 'actual_values'\n",
    "mae = mean_absolute_error(y_test, predicted_values)\n",
    "mse = mean_squared_error(y_test, predicted_values)\n",
    "rmse = np.sqrt(mse)\n",
    "r2 = r2_score(y_test, predicted_values)\n",
    "print(\"MSE:\", mse)\n",
    "print(\"MAE:\", mae)\n",
    "print(\"RMSE:\", rmse)\n",
    "print(\"R2 Score:\", r2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a0b5654d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59916 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.plot(t, series[0:,:,0],color='gray',marker='o')\n",
    "plt.plot(x_train_data,train_data[0:,:,0],'o')\n",
    "plt.plot(x_test_data,test_data[:,:,0],'g.')\n",
    "plt.plot(x_test_data,predicted_values,'r*')\n",
    "\n",
    "\n",
    "plt.xlabel('time')\n",
    "plt.ylabel('Value')\n",
    "plt.title('Time Series Data')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ecac9d01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "100\n",
      "80\n",
      "80\n",
      "20\n",
      "20\n",
      "20\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fe6152d0b50>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 76516 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = series[0:,:,0]\n",
    "y_train = train_data[0:,:,0]\n",
    "y_test = test_data[0:,:,0]\n",
    "\n",
    "print(len(data))\n",
    "print(len(t))\n",
    "print(len(y_train))\n",
    "print(len(x_train_data))\n",
    "print(len(y_test))\n",
    "print(len(x_test_data))\n",
    "print(len(predicted_values))\n",
    "\n",
    "\n",
    "#plt.plot(t,data)\n",
    "plt.plot(x_train_data,y_train)\n",
    "plt.plot(x_test_data,y_test)\n",
    "plt.plot(t,data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "770832f3",
   "metadata": {},
   "source": [
    "# ARIMA\n",
    "**Now applying the Autoregressive Integrated Moving Average Model** \n",
    " - (1,0,0) = AR(1) purely Autoregressive with phase 1 autocorrelation\n",
    " - (0,1,0) = I(1) purely Integrated\n",
    " - (0,0,1) = MA(1) purely Moving Average with window 1\n",
    " - (0,0,0) = White Noise\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ccfe85b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from statsmodels.tsa.arima.model import ARIMA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0268b937",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ARIMA model (p=0,d=0,q=0)  , RMSE =6.46 % forecast: 3 steps\n",
      "ARIMA model (p=1,d=0,q=0)  , RMSE =32.88 % forecast: 3 steps\n",
      "ARIMA model (p=2,d=0,q=0)  , RMSE =32.69 % forecast: 3 steps\n",
      "ARIMA model (p=3,d=0,q=0)  , RMSE =32.47 % forecast: 3 steps\n",
      "ARIMA model (p=4,d=0,q=0)  , RMSE =32.32 % forecast: 3 steps\n",
      "ARIMA model (p=5,d=0,q=0)  , RMSE =31.70 % forecast: 3 steps\n",
      "ARIMA model (p=6,d=0,q=0)  , RMSE =30.88 % forecast: 3 steps\n",
      "ARIMA model (p=7,d=0,q=0)  , RMSE =29.62 % forecast: 3 steps\n",
      "ARIMA model (p=0,d=1,q=0)  , RMSE =35.65 % forecast: 3 steps\n",
      "ARIMA model (p=1,d=1,q=0)  , RMSE =33.10 % forecast: 3 steps\n",
      "ARIMA model (p=2,d=1,q=0)  , RMSE =33.35 % forecast: 3 steps\n",
      "ARIMA model (p=3,d=1,q=0)  , RMSE =33.23 % forecast: 3 steps\n",
      "ARIMA model (p=4,d=1,q=0)  , RMSE =35.57 % forecast: 3 steps\n",
      "ARIMA model (p=5,d=1,q=0)  , RMSE =37.87 % forecast: 3 steps\n",
      "ARIMA model (p=6,d=1,q=0)  , RMSE =37.94 % forecast: 3 steps\n",
      "ARIMA model (p=7,d=1,q=0)  , RMSE =38.64 % forecast: 3 steps\n",
      "ARIMA model (p=0,d=2,q=0)  , RMSE =38.57 % forecast: 3 steps\n",
      "ARIMA model (p=1,d=2,q=0)  , RMSE =30.03 % forecast: 3 steps\n",
      "ARIMA model (p=2,d=2,q=0)  , RMSE =31.82 % forecast: 3 steps\n",
      "ARIMA model (p=3,d=2,q=0)  , RMSE =32.04 % forecast: 3 steps\n",
      "ARIMA model (p=4,d=2,q=0)  , RMSE =28.87 % forecast: 3 steps\n",
      "ARIMA model (p=5,d=2,q=0)  , RMSE =40.60 % forecast: 3 steps\n",
      "ARIMA model (p=6,d=2,q=0)  , RMSE =46.29 % forecast: 3 steps\n",
      "ARIMA model (p=7,d=2,q=0)  , RMSE =40.32 % forecast: 3 steps\n",
      "ARIMA model (p=0,d=3,q=0)  , RMSE =41.76 % forecast: 3 steps\n",
      "ARIMA model (p=1,d=3,q=0)  , RMSE =47.62 % forecast: 3 steps\n",
      "ARIMA model (p=2,d=3,q=0)  , RMSE =31.30 % forecast: 3 steps\n",
      "ARIMA model (p=3,d=3,q=0)  , RMSE =59.52 % forecast: 3 steps\n",
      "ARIMA model (p=4,d=3,q=0)  , RMSE =44.26 % forecast: 3 steps\n",
      "====================================================================================================\n",
      "II. RMSE Minimum criterion (<28.87 %) reached. Stopping the loop.\n",
      "ARIMA model (p=4,d=3,q=0)  , RMSE =15.47 % forecast: 3 steps\n",
      "====================================================================================================\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 116512 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 46428 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80\n",
      "20\n",
      "100\n",
      "Time used: 0.90 seconds\n"
     ]
    }
   ],
   "source": [
    "# Plot the combined_data\n",
    "import time\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "data = series[0:,:,0]\n",
    "y_train = train_data[0:,:,0]\n",
    "y_test = test_data[0:,:,0]\n",
    "mse_pqd = []\n",
    "modes = []\n",
    "convergence = 4.99\n",
    "\n",
    "n_forecast_steps = 3\n",
    "x_forecast = x_test_data[0:n_forecast_steps]\n",
    "\n",
    "convergence_reached = False  # Initialize convergence flag\n",
    "\n",
    "while not convergence_reached:\n",
    "\n",
    "    # Run the loop as far as the convergence criterion is fulfilled\n",
    "   \n",
    "    \n",
    "    # Check convergence criterion\n",
    "\n",
    "    for q in range(8):\n",
    "        for d in range(8):\n",
    "            for p in range(8):\n",
    "                \n",
    "                model = ARIMA(y_train, order=(p, d, q))  # Replace p, d, q with appropriate values\n",
    "                model_fit = model.fit()\n",
    "\n",
    "                forecast = model_fit.forecast(steps=n_forecast_steps)  # Replace 3 with the number of steps you want to forecast\n",
    "               \n",
    "                print(f'ARIMA model (p={p},d={d},q={q})  , RMSE ={mse:.2f} % forecast: {n_forecast_steps} steps')\n",
    "\n",
    "                mse = 100*np.sqrt(mean_squared_error(y_test[0:n_forecast_steps], forecast))/abs(np.mean(forecast))\n",
    "                modes.append(f'({p},{d},{q})')\n",
    "                \n",
    "                if len(mse_pqd) > 20:\n",
    "                    min_rmse = np.min(mse_pqd[1:])\n",
    "                    if mse < min_rmse*1.05:\n",
    "                        print(\"=\"*100)\n",
    "                        print(f\"II. RMSE Minimum criterion (<{min_rmse:.2f} %) reached. Stopping the loop.\")\n",
    "                        convergence_reached = True  # Set convergence flag\n",
    "                        mse_pqd.append(mse)\n",
    "                        print(f'ARIMA model (p={p},d={d},q={q})  , RMSE ={mse:.2f} % forecast: {n_forecast_steps} steps')\n",
    "                        print(\"=\"*100)\n",
    "                        break\n",
    "                    \n",
    "                mse_pqd.append(mse)\n",
    "                \n",
    "               \n",
    "                if mse < convergence:\n",
    "                    print(\"=\"*100)\n",
    "                    print(f\"I. RMSE Convergence criterion (<{convergence} %)  reached. Stopping the loop.\")\n",
    "                    print(f'ARIMA model (p={p},d={d},q={q})  , RMSE ={mse:.2f} % forecast: {n_forecast_steps} steps')\n",
    "                    print(\"=\"*100)\n",
    "                    convergence_reached = True  # Set convergence flag\n",
    "                    break\n",
    "            if convergence_reached:  # Check if convergence flag is set\n",
    "                break\n",
    "        if convergence_reached:  # Check if convergence flag is set\n",
    "            break\n",
    "    if convergence_reached:  # Check if convergence flag is set\n",
    "        break\n",
    "                \n",
    "           \n",
    "# Create a fixed plot\n",
    "plt.figure(figsize=(8, 6))               \n",
    "plt.plot(t, data,color='gray',label='Original')\n",
    "plt.plot(x_train_data, y_train,label='Train')\n",
    "plt.plot(x_test_data, y_test,label='Test',color='green')\n",
    "plt.plot(x_forecast, forecast,'r*',label='Forecast')\n",
    "plt.xlabel('Index')\n",
    "plt.ylabel('Value')\n",
    "plt.legend()\n",
    "plt.title(f'ARIMA model (p={p},d={d},q={q})  , RMSE={mse:.2f} % forecast: {n_forecast_steps: 2d} next steps ')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "# Create a fixed plot\n",
    "plt.figure(figsize=(8, 6))\n",
    "xx = np.linspace(0,100,len(mse_pqd))\n",
    "plt.plot(modes, mse_pqd)\n",
    "plt.grid(True)\n",
    "plt.ylabel('MSE (log scale)')\n",
    "plt.yscale('log')  # Set y-axis scale to logarithmic\n",
    "plt.xticks(modes[::5],rotation=90)\n",
    "plt.title(f'ARIMA {modes[np.argmin(mse_pqd)]} with RMSE = {np.min(mse_pqd):.2f} %')\n",
    "plt.text(np.argmin(mse_pqd),np.min(mse_pqd),'x',color='r')\n",
    "plt.show()\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "time_used = end_time - start_time\n",
    "print(len(x_train_data))\n",
    "print(len(x_test_data))\n",
    "print(len(x))\n",
    "print(\"Time used:\", f'{time_used:.2f}', \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "901da521",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(x_train_data),len(y_train_data)  80 80\n",
      "len(x_test_data),len(y_test_data)  20 20\n",
      "len(x),len(t)  100 100\n"
     ]
    }
   ],
   "source": [
    "print(\"len(x_train_data),len(y_train_data) \",np.size(x_train_data),np.size(y_train))\n",
    "print(\"len(x_test_data),len(y_test_data) \",np.size(x_test_data),np.size(y_test))\n",
    "print(\"len(x),len(t) \",np.size(x),np.size(t))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99d86e04",
   "metadata": {},
   "source": [
    "# LSTM\n",
    "**Using the Long Short Term Memory for prediction** \n",
    "- Define the LSTM model architecture\n",
    "- model = Sequential() For timeseries the Sequential Keras class is suitable\n",
    "- model.add(LSTM(64, input_shape=(time_steps, 1))) Add LSTM cell into the Sequential architectur\n",
    "- model.add(Dropout(0.2))  # Add dropout layer with a dropout rate of 0.2\n",
    "- model.add(Dense(1)) Predict the next step of the time series\n",
    "- model.compile(optimizer='adam', loss='mean_squared_error') At the end the model will be compiled with an Adam optimizer (For Gradient descent to reach the cost function) and the loss (cost function) of Mean Square Error (Similar to Least Square Fit).\n",
    "\n",
    "# Train the model\n",
    " - model.fit(X_train, y_train, epochs=50, batch_size=16) As model is compiled, then a fit can be performed on the training data set with 50 epochs (runs for convergence) taking 16 data points as batches (at once) for running moving averags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3142fd3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-07 18:24:03.243599: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-03-07 18:24:03.246569: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-03-07 18:24:03.297801: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-03-07 18:24:03.298217: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-03-07 18:24:04.020882: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense, Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5eeb24e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values[:4] [[-6.994966 ]\n",
      " [-7.772627 ]\n",
      " [-5.3947926]\n",
      " [-3.7644155]]\n",
      "len(values) 100\n",
      "len(y_train) 80\n",
      "scaler MinMaxScaler()\n",
      "normalized_values[:4] [[0.50778383]\n",
      " [0.44069958]\n",
      " [0.6458214 ]\n",
      " [0.78646445]]\n"
     ]
    }
   ],
   "source": [
    "# Assuming you have the time series data stored in a DataFrame called 'data'\n",
    "\n",
    "# Extract the values from the 'value' column as a NumPy array\n",
    "values = data#y_train #data['value'].values\n",
    "print('values[:4]',values[:4])\n",
    "print(\"len(values)\",len(values))\n",
    "print(\"len(y_train)\",len(y_train))\n",
    "\n",
    "\n",
    "\n",
    "# Normalize the values to a range between 0 and 1\n",
    "scaler = MinMaxScaler()\n",
    "print('scaler',scaler)\n",
    "normalized_values = scaler.fit_transform(values.reshape(-1, 1))\n",
    "print('normalized_values[:4]',normalized_values[:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e3e4cd76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80,)\n",
      "(20,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fe5c63e2040>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 61384 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Define the size of the training set\n",
    "train_size = int(len(normalized_values) * 0.8)  # 80% for training\n",
    "\n",
    "# Split the data into training and testing sets\n",
    "train_data = normalized_values[:train_size,0]\n",
    "test_data = normalized_values[train_size:,0]\n",
    "\n",
    "print(np.shape(train_data))\n",
    "print(np.shape(test_data))\n",
    "plt.plot(train_data)\n",
    "plt.plot(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f3bcb643",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the number of previous time steps to consider for each prediction\n",
    "time_steps = 5\n",
    "\n",
    "# Create input and output sequences for training and testing\n",
    "def create_sequences(data, time_steps):\n",
    "    X = []\n",
    "    y = []\n",
    "    for i in range(len(data) - time_steps):\n",
    "        \n",
    "        X.append(data[i:i + time_steps])\n",
    "        y.append(data[i + time_steps])\n",
    "      \n",
    "    return np.array(X), np.array(y)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cfd0c4a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print('train_data[0:5] ',train_data[0:5])\n",
    "#print('train_data[5] ',train_data[5])\n",
    "#print('X_train[0:21] ',X_train[0:3,:])\n",
    "#print('y_train[0:21]',y_train[0:3:])\n",
    "#print('X_test[:10]',X_test[:],0)\n",
    "#print('y_test[:10] ',y_test[:])\n",
    "#print('len(X_train),len(y_train)',len(X_train),len(y_train))\n",
    "#print('len(X_test),len(y_test)',len(X_test),len(y_test))\n",
    "#print('type(X_train)',type(X_train))\n",
    "#print('type(X_test)',type(X_test))\n",
    "#print(np.shape(X_train))\n",
    "#print(np.shape(y_train))\n",
    "#print(np.shape(X_test))\n",
    "#print(np.shape(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4a0f5707",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Create sequences for training and testing\n",
    "X_train, y_train = create_sequences(train_data, time_steps)\n",
    "X_test, y_test = create_sequences(test_data, time_steps)\n",
    "\n",
    "\n",
    "# Define the Sequential model architecture with LSTM Input cell\n",
    "model = Sequential()\n",
    "model.add(LSTM(64, input_shape=(time_steps, 1)))\n",
    "#model.add(Dropout(0.4))  # Add dropout layer with a dropout rate of 0.2\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adam', loss='mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b4b5a63a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "5/5 [==============================] - 1s 2ms/step - loss: 0.3204\n",
      "Epoch 2/20\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.2087\n",
      "Epoch 3/20\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1228\n",
      "Epoch 4/20\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.0581\n",
      "Epoch 5/20\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.0454\n",
      "Epoch 6/20\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.0521\n",
      "Epoch 7/20\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.0479\n",
      "Epoch 8/20\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.0412\n",
      "Epoch 9/20\n",
      "5/5 [==============================] - 0s 6ms/step - loss: 0.0412\n",
      "Epoch 10/20\n",
      "5/5 [==============================] - 0s 6ms/step - loss: 0.0417\n",
      "Epoch 11/20\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.0408\n",
      "Epoch 12/20\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.0400\n",
      "Epoch 13/20\n",
      "5/5 [==============================] - 0s 8ms/step - loss: 0.0401\n",
      "Epoch 14/20\n",
      "5/5 [==============================] - 0s 8ms/step - loss: 0.0401\n",
      "Epoch 15/20\n",
      "5/5 [==============================] - 0s 8ms/step - loss: 0.0402\n",
      "Epoch 16/20\n",
      "5/5 [==============================] - 0s 9ms/step - loss: 0.0397\n",
      "Epoch 17/20\n",
      "5/5 [==============================] - 0s 6ms/step - loss: 0.0395\n",
      "Epoch 18/20\n",
      "5/5 [==============================] - 0s 7ms/step - loss: 0.0394\n",
      "Epoch 19/20\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.0394\n",
      "Epoch 20/20\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.0393\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 26344 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Train the model and store the training history\n",
    "history = model.fit(X_train, y_train, epochs=20, batch_size=16)\n",
    "\n",
    "# Access the loss values from the history object\n",
    "loss = history.history['loss']\n",
    "#acc = history.history['accuracy']\n",
    "# Plot the loss curve\n",
    "plt.plot(range(len(loss)), loss)\n",
    "#plt.plot(range(len(loss)), acc)\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Training Loss vs. Epochs')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1166e6db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 219ms/step\n",
      "(15, 1) (15, 1)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 47004 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean y_test -6.6228986\n",
      "MSE : 7.691864\n",
      "MAE : 2.163792\n",
      "RMSE : 2.773421\n",
      "MSE %: -116.14044904708862\n",
      "MAE %: -32.67137408256531\n",
      "RMSE %: -41.87624156475067\n"
     ]
    }
   ],
   "source": [
    "# Generate predictions on the test data\n",
    "y_pred = model.predict(X_test)\n",
    "#print('X_test',X_test)\n",
    "#print('y_pred',y_pred)\n",
    "# Inverse transform the normalized predictions to the original scale\n",
    "y_pred = scaler.inverse_transform(y_pred)\n",
    "\n",
    "\n",
    "\n",
    "# Inverse transform the normalized test data to the original scale\n",
    "y_test = scaler.inverse_transform(y_test.reshape(-1, 1))\n",
    "print(np.shape(y_test),np.shape(y_pred))\n",
    "\n",
    "\n",
    "\n",
    "# Plot the actual values and the predictions\n",
    "plt.plot(y_test, label='Test sequence',linestyle='dotted')\n",
    "plt.plot(y_pred, label='Predicted sequence')\n",
    "plt.xlabel('Time')\n",
    "plt.ylabel('Value')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "mae = mean_absolute_error(y_test, y_pred)\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "rmse = np.sqrt(mse)\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "print(\"mean y_test\", np.mean(y_test))\n",
    "print(\"MSE :\", mse)\n",
    "print(\"MAE :\", mae)\n",
    "print(\"RMSE :\", rmse)\n",
    "\n",
    "\n",
    "print(\"MSE %:\", mse/np.mean(y_test)*100)\n",
    "print(\"MAE %:\", mae/np.mean(y_test)*100)\n",
    "print(\"RMSE %:\", rmse/np.mean(y_test)*100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e8fe15a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15ff3523",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
