Notebooks/TensorFlow/ANN/TF_ANN-II.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"color:rgb(0,120,170)\">Neural Networks and Deep Learning</h1>\n",
    "<h2 style=\"color:rgb(0,120,170)\">Intro to Tensor Flow and Keras</h2>\n",
    "<h3 style=\"color:rgb(0,120,170)\">3 ways to create a Machine Learning model with Keras and TensorFlow 2.0</h3>\n",
    "\n",
    "[Source](https://medium.com/@bindiatwork/3-ways-to-create-a-machine-learning-model-with-keras-and-tensorflow-2-0-de09323af4d3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-22 00:24:04.909576: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-10-22 00:24:04.925480: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-10-22 00:24:04.930219: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-10-22 00:24:04.942270: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-10-22 00:24:05.709107: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "iris = load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.17.0\n",
      "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1729549447.155900   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1729549447.193484   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1729549447.193680   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)\n",
    "try:\n",
    "    physical_devices = tf.config.list_physical_devices('GPU') \n",
    "    #tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "    for device in physical_devices:\n",
    "        tf.config.experimental.set_memory_growth(device, True)\n",
    "    print(physical_devices)\n",
    "except:\n",
    "    print(\"No GPU\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploring Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data into a DataFrame\n",
    "df = pd.DataFrame(iris.data, columns=iris.feature_names)\n",
    "# Convert datatype to float\n",
    "df = df.astype(float)\n",
    "# append \"target\" and name it \"label\"\n",
    "df['label'] = iris.target\n",
    "# Use string label instead\n",
    "df['label'] = df.label.replace(dict(enumerate(iris.target_names)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "0                5.1               3.5                1.4               0.2   \n",
       "1                4.9               3.0                1.4               0.2   \n",
       "2                4.7               3.2                1.3               0.2   \n",
       "3                4.6               3.1                1.5               0.2   \n",
       "4                5.0               3.6                1.4               0.2   \n",
       "\n",
       "    label  \n",
       "0  setosa  \n",
       "1  setosa  \n",
       "2  setosa  \n",
       "3  setosa  \n",
       "4  setosa  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['setosa', 'versicolor', 'virginica'], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.label.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing data for Keras and TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label -> one-hot encoding\n",
    "label = pd.get_dummies(df['label'])\n",
    "label.columns = ['label_' + str(x) for x in label.columns]\n",
    "df = pd.concat([df, label], axis=1)\n",
    "\n",
    "# drop old label\n",
    "df.drop(['label'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>label_setosa</th>\n",
       "      <th>label_versicolor</th>\n",
       "      <th>label_virginica</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "0                5.1               3.5                1.4               0.2   \n",
       "1                4.9               3.0                1.4               0.2   \n",
       "2                4.7               3.2                1.3               0.2   \n",
       "3                4.6               3.1                1.5               0.2   \n",
       "4                5.0               3.6                1.4               0.2   \n",
       "\n",
       "   label_setosa  label_versicolor  label_virginica  \n",
       "0          True             False            False  \n",
       "1          True             False            False  \n",
       "2          True             False            False  \n",
       "3          True             False            False  \n",
       "4          True             False            False  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating X and y\n",
    "\n",
    "X = df[['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']]\n",
    "# Convert DataFrame into np array\n",
    "X = np.asarray(X)\n",
    "\n",
    "y = df[['label_setosa', 'label_versicolor', 'label_virginica']]\n",
    "# Convert DataFrame into np array\n",
    "y = np.asarray(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data set in a training set (75%) and a test set (25%)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4.8, 3.4, 1.9, 0.2])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Sequential model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/renato/Documents/default-env/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n",
      "I0000 00:00:1729549447.308851   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1729549447.309110   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1729549447.309259   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1729549447.377352   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1729549447.377558   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1729549447.377699   25573 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-10-22 00:24:07.377810: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 10407 MB memory:  -> device: 0, name: NVIDIA TITAN Xp, pci bus id: 0000:01:00.0, compute capability: 6.1\n",
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "W0000 00:00:1729549447.565389   25736 gpu_backend_lib.cc:593] Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice. This may result in compilation or runtime failures, if the program we try to run uses routines from libdevice.\n",
      "Searched for CUDA in the following directories:\n",
      "  ./cuda_sdk_lib\n",
      "  /usr/local/cuda-12.3\n",
      "  /usr/local/cuda\n",
      "  /home/renato/Documents/default-env/lib/python3.12/site-packages/tensorflow/python/platform/../../../nvidia/cuda_nvcc\n",
      "  /home/renato/Documents/default-env/lib/python3.12/site-packages/tensorflow/python/platform/../../../../nvidia/cuda_nvcc\n",
      "  .\n",
      "You can choose the search directory by setting xla_gpu_cuda_data_dir in HloModule's DebugOptions.  For most apps, setting the environment variable XLA_FLAGS=--xla_gpu_cuda_data_dir=/path/to/cuda will work.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"sequential\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5</span>)              │            <span style=\"color: #00af00; text-decoration-color: #00af00\">25</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">10</span>)             │            <span style=\"color: #00af00; text-decoration-color: #00af00\">60</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">3</span>)              │            <span style=\"color: #00af00; text-decoration-color: #00af00\">33</span> │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ dense (\u001b[38;5;33mDense\u001b[0m)                   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5\u001b[0m)              │            \u001b[38;5;34m25\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_1 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m10\u001b[0m)             │            \u001b[38;5;34m60\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_2 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m3\u001b[0m)              │            \u001b[38;5;34m33\u001b[0m │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">118</span> (472.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m118\u001b[0m (472.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">118</span> (472.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m118\u001b[0m (472.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A) Passing a list of layers to the constructor\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "model = Sequential([Dense(5, activation='relu', input_shape=(4,)),\n",
    "                    Dense(10, activation='relu'),\n",
    "                    Dense(3, activation='softmax'),\n",
    "                   ])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential_1\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"sequential_1\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ dense_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5</span>)              │            <span style=\"color: #00af00; text-decoration-color: #00af00\">25</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">10</span>)             │            <span style=\"color: #00af00; text-decoration-color: #00af00\">60</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">3</span>)              │            <span style=\"color: #00af00; text-decoration-color: #00af00\">33</span> │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ dense_3 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5\u001b[0m)              │            \u001b[38;5;34m25\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_4 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m10\u001b[0m)             │            \u001b[38;5;34m60\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense_5 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m3\u001b[0m)              │            \u001b[38;5;34m33\u001b[0m │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">118</span> (472.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m118\u001b[0m (472.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">118</span> (472.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m118\u001b[0m (472.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# B) Adding layer via add() method\n",
    "\n",
    "model_seq = Sequential()\n",
    "model_seq.add(Dense(5, activation='relu', input_shape=(4,)))\n",
    "model_seq.add(Dense(10, activation='relu'))\n",
    "model_seq.add(Dense(3, activation='softmax'))\n",
    "\n",
    "model_seq.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Training a Sequential model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_seq.compile(optimizer='rmsprop', \n",
    "                  loss='categorical_crossentropy', \n",
    "                  metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0000 00:00:1729549448.615093   25728 service.cc:146] XLA service 0x7dce90008190 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "I0000 00:00:1729549448.615114   25728 service.cc:154]   StreamExecutor device (0): NVIDIA TITAN Xp, Compute Capability 6.1\n",
      "2024-10-22 00:24:08.634011: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-10-22 00:24:08.700750: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n",
      "W0000 00:00:1729549448.954995   25728 gpu_backend_lib.cc:631] libdevice is required by this HLO module but was not found at ./libdevice.10.bc\n",
      "2024-10-22 00:24:08.957449: W tensorflow/core/framework/op_kernel.cc:1840] OP_REQUIRES failed at xla_ops.cc:577 : INTERNAL: libdevice not found at ./libdevice.10.bc\n",
      "2024-10-22 00:24:08.957484: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: INTERNAL: libdevice not found at ./libdevice.10.bc\n",
      "\t [[{{node StatefulPartitionedCall}}]]\n"
     ]
    },
    {
     "ename": "InternalError",
     "evalue": "Graph execution error:\n\nDetected at node StatefulPartitionedCall defined at (most recent call last):\n  File \"<frozen runpy>\", line 198, in _run_module_as_main\n\n  File \"<frozen runpy>\", line 88, in _run_code\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel_launcher.py\", line 18, in <module>\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/traitlets/config/application.py\", line 1075, in launch_instance\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelapp.py\", line 739, in start\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/tornado/platform/asyncio.py\", line 205, in start\n\n  File \"/usr/lib/python3.12/asyncio/base_events.py\", line 641, in run_forever\n\n  File \"/usr/lib/python3.12/asyncio/base_events.py\", line 1987, in _run_once\n\n  File \"/usr/lib/python3.12/asyncio/events.py\", line 88, in _run\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 545, in dispatch_queue\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 534, in process_one\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 437, in dispatch_shell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 362, in execute_request\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 778, in execute_request\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 449, in do_execute\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/zmqshell.py\", line 549, in run_cell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3075, in run_cell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3130, in _run_cell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/async_helpers.py\", line 128, in _pseudo_sync_runner\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3334, in run_cell_async\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3517, in run_ast_nodes\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3577, in run_code\n\n  File \"/tmp/ipykernel_25573/3603885880.py\", line 1, in <module>\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py\", line 117, in error_handler\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/keras/src/backend/tensorflow/trainer.py\", line 320, in fit\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/keras/src/backend/tensorflow/trainer.py\", line 121, in one_step_on_iterator\n\nlibdevice not found at ./libdevice.10.bc\n\t [[{{node StatefulPartitionedCall}}]] [Op:__inference_one_step_on_iterator_1018]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInternalError\u001b[0m                             Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[14], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m history \u001b[38;5;241m=\u001b[39m \u001b[43mmodel_seq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      2\u001b[0m \u001b[43m                    \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m64\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m                    \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m30\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m                    \u001b[49m\u001b[43mvalidation_split\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.2\u001b[39;49m\n\u001b[1;32m      5\u001b[0m \u001b[43m                   \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Documents/default-env/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py:122\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    119\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m    120\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m    121\u001b[0m     \u001b[38;5;66;03m# `keras.config.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m--> 122\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    123\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    124\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m~/Documents/default-env/lib/python3.12/site-packages/tensorflow/python/eager/execute.py:53\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     52\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 53\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m pywrap_tfe\u001b[38;5;241m.\u001b[39mTFE_Py_Execute(ctx\u001b[38;5;241m.\u001b[39m_handle, device_name, op_name,\n\u001b[1;32m     54\u001b[0m                                       inputs, attrs, num_outputs)\n\u001b[1;32m     55\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     56\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mInternalError\u001b[0m: Graph execution error:\n\nDetected at node StatefulPartitionedCall defined at (most recent call last):\n  File \"<frozen runpy>\", line 198, in _run_module_as_main\n\n  File \"<frozen runpy>\", line 88, in _run_code\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel_launcher.py\", line 18, in <module>\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/traitlets/config/application.py\", line 1075, in launch_instance\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelapp.py\", line 739, in start\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/tornado/platform/asyncio.py\", line 205, in start\n\n  File \"/usr/lib/python3.12/asyncio/base_events.py\", line 641, in run_forever\n\n  File \"/usr/lib/python3.12/asyncio/base_events.py\", line 1987, in _run_once\n\n  File \"/usr/lib/python3.12/asyncio/events.py\", line 88, in _run\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 545, in dispatch_queue\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 534, in process_one\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 437, in dispatch_shell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 362, in execute_request\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 778, in execute_request\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 449, in do_execute\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/ipykernel/zmqshell.py\", line 549, in run_cell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3075, in run_cell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3130, in _run_cell\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/async_helpers.py\", line 128, in _pseudo_sync_runner\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3334, in run_cell_async\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3517, in run_ast_nodes\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3577, in run_code\n\n  File \"/tmp/ipykernel_25573/3603885880.py\", line 1, in <module>\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py\", line 117, in error_handler\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/keras/src/backend/tensorflow/trainer.py\", line 320, in fit\n\n  File \"/home/renato/Documents/default-env/lib/python3.12/site-packages/keras/src/backend/tensorflow/trainer.py\", line 121, in one_step_on_iterator\n\nlibdevice not found at ./libdevice.10.bc\n\t [[{{node StatefulPartitionedCall}}]] [Op:__inference_one_step_on_iterator_1018]"
     ]
    }
   ],
   "source": [
    "history = model_seq.fit(X_train, y_train,\n",
    "                    batch_size= 64,\n",
    "                    epochs= 30,\n",
    "                    validation_split=0.2\n",
    "                   )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'svg'\n",
    "def plot_metric(history, metric):\n",
    "    train_metrics = history.history[metric]\n",
    "    val_metrics = history.history['val_'+metric]\n",
    "    epochs = range(1, len(train_metrics) + 1)\n",
    "    plt.plot(epochs, train_metrics, 'bo--')\n",
    "    plt.plot(epochs, val_metrics, 'ro-')\n",
    "    plt.title('Training and validation '+ metric)\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(metric)\n",
    "    plt.legend([\"train_\"+metric, 'val_'+metric])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metric(history, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metric(history, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_seq.evaluate(x = X_test,y = y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Functional API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, Dense\n",
    "\n",
    "\n",
    "# This returns a tensor\n",
    "inputs = Input(shape=(4,))\n",
    "\n",
    "# a layer instance is callable on a tensor, and returns a tensor\n",
    "x = Dense(5, activation='relu')(inputs)\n",
    "x = Dense(10, activation='relu')(x)\n",
    "outputs = Dense(3, activation='softmax')(x)\n",
    "\n",
    "# This creates a model that includes\n",
    "# the Input layer and three Dense layers\n",
    "model_fun_api = Model(inputs=inputs, outputs=outputs)\n",
    "\n",
    "model_fun_api.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Training a Function API model\n",
    "Such model can be trained just like Keras Sequential model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First configure model using model.compile()\n",
    "model_fun_api.compile(optimizer='rmsprop', \n",
    "                      loss='categorical_crossentropy', \n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "# Then, train the model with fit()\n",
    "history = model_fun_api.fit(X_train, y_train,\n",
    "                    batch_size= 64,\n",
    "                    epochs= 30,\n",
    "                    validation_split=0.2\n",
    "                   )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metric(history, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metric(history, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_fun_api.evaluate(x = X_test,y = y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 Multi-input and multi-output model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For the scenario with 2 inputs and 1 output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import concatenate\n",
    "\n",
    "# 2 inputs, one structured data, the other image data\n",
    "input_structure = Input(shape=(4,), name='input_structure')\n",
    "input_image = Input(shape=(256,), name='input_image')\n",
    "\n",
    "# middle layers\n",
    "x_1 = Dense(10, activation='relu')(input_structure)\n",
    "x_2 = Dense(100, activation='relu')(input_image)\n",
    "\n",
    "c = concatenate([x_1, x_2])\n",
    "\n",
    "outputs = Dense(3, activation='sigmoid', name='outputs')(c)\n",
    "\n",
    "model = Model(inputs=[input_structure, input_image], outputs=outputs)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For the scenario with 1 input and 2 outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 1 inputs, 2 output\n",
    "\n",
    "from tensorflow.keras.layers import concatenate\n",
    "\n",
    "# only one input\n",
    "input_image = Input(shape=(256,), name='input_image')\n",
    "\n",
    "# middle layer\n",
    "x = Dense(300, activation='relu')(input_image)\n",
    "\n",
    "# output layer\n",
    "output_1 = Dense(1, activation='sigmoid', name='output_1')(x)\n",
    "output_2 = Dense(3, activation='softmax', name='output_2')(x)\n",
    "\n",
    "model = Model(inputs=input_image, outputs=[output_1, output_2])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For the scenario with 2 inputs and 2 outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 2 inputs, 2 outputs\n",
    "from tensorflow.keras.layers import concatenate\n",
    "\n",
    "# 2 inputs, one structured data, the other image data\n",
    "input_structured = Input(shape=(4,), name='input_structured')\n",
    "input_image = Input(shape=(256,), name='input_image')\n",
    "\n",
    "# middle layers\n",
    "x_1 = Dense(10, activation='relu')(input_structured)\n",
    "x_2 = Dense(300, activation='relu')(input_image)\n",
    "\n",
    "c = concatenate([x_1, x_2])\n",
    "\n",
    "# output layser\n",
    "output_1 = Dense(1, activation='sigmoid', name='output_1')(c)\n",
    "output_2 = Dense(3, activation='softmax', name='output_2')(c)\n",
    "\n",
    "model = Model(inputs=[input_structured, input_image], outputs=[output_1, output_2])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 Shared layer in Functional API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = Input(shape=(4,))\n",
    "\n",
    "# a layer instance is callable on a tensor, and returns a tensor\n",
    "shared_layer = Dense(4, activation='relu')\n",
    "\n",
    "# Now we apply the layer three times\n",
    "x = shared_layer(inputs)\n",
    "x = shared_layer(x)\n",
    "x = shared_layer(x)\n",
    "x = shared_layer(x)\n",
    "\n",
    "predictions = Dense(3, activation='softmax')(x)\n",
    "\n",
    "model = Model(inputs=inputs, outputs=predictions)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Model Subclassing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "class CustomModel(Model):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(CustomModel, self).__init__(**kwargs)\n",
    "        self.dense1 = Dense(5, activation='relu', )\n",
    "        self.dense2 = Dense(10, activation='relu')\n",
    "        self.dense3 = Dense(3, activation='softmax')\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        x = self.dense1(inputs)\n",
    "        x = self.dense2(x)\n",
    "        return self.dense3(x)\n",
    "\n",
    "my_custom_model = CustomModel(name='my_custom_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Training a Model Subclassing model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_custom_model.compile(optimizer='rmsprop', \n",
    "                        loss='categorical_crossentropy', \n",
    "                        metrics=['accuracy'])\n",
    "\n",
    "history = my_custom_model.fit(X_train, y_train, \n",
    "                              batch_size= 64,\n",
    "                              epochs= 30,\n",
    "                              validation_split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_custom_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metric(history, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metric(history, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_custom_model.evaluate(x = X_test,y = y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}