Notebooks/TensorFlow/ANN/TF_ANN-IV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "ed4cd7c86887522b8c8d4b8c3c53ba1027949f9b"
   },
   "source": [
    "<h1 style=\"color:rgb(0,120,170)\">Neural Networks and Deep Learning</h1>\n",
    "<h2 style=\"color:rgb(0,120,170)\">Intro to Tensor Flow and Keras</h2>\n",
    "<h2 style=\"color:rgb(0,120,170)\">Testing model hiperparameters</h2>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!pip install -U tensorflow tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "#import re\n",
    "import time\n",
    "import datetime\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style(\"darkgrid\")\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorboard.plugins.hparams import api_pb2\n",
    "from tensorboard.plugins.hparams import summary as hparams_summary\n",
    "%load_ext tensorboard\n",
    "\n",
    "from google.protobuf import struct_pb2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.8.0\n",
      "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-02-19 10:18:41.798402: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:41.826584: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:41.826739: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)\n",
    "try:\n",
    "    physical_devices = tf.config.list_physical_devices('GPU') \n",
    "    #tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "    for device in physical_devices:\n",
    "        tf.config.experimental.set_memory_growth(device, True)\n",
    "    print(physical_devices)\n",
    "except:\n",
    "    print(\"No GPU\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Conv1D.xlsx', 'best_model.h5', 'Conv2D.xlsx', 'best_parameters_svm.pickle', 'UK_LSOA_bounding_boxes.csv', 'kaggle_titanic_test.csv', 'Concrete_Data.xlsx', 'winequality-red.csv', 'London_accidents_merged.csv', 'Population_density.csv', 'tweets.csv.gz', 'Traffic_averages.csv', 'Conv3D.xlsx', 'gender_submission.csv', 'pima-indians-diabetes.csv', 'model3_safe_dataset.csv', 'wonderland.txt', 'ion.csv', 'kaggle_digits_train.zip', 'MNIST', 'kaggle_titanic_train.csv', 'kaggle_digits_test.zip', 'kaggle_digits_sample_submission.csv', 'titanic.jpeg', 'airbnb.csv']\n"
     ]
    }
   ],
   "source": [
    "print(os.listdir(\"../../../data\"))\n",
    "\n",
    "# Clear any logs from previous runs\n",
    "#!rm -rf ./logs/ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1st - [Concrete data](https://medium.com/@rahul.raoniar/hyperparameter-tuning-of-keras-deep-learning-model-in-python-e426c94a43d7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense, BatchNormalization, Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.wrappers.scikit_learn import KerasRegressor\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras import regularizers\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import RandomizedSearchCV, KFold, cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cement</th>\n",
       "      <th>BF_slag</th>\n",
       "      <th>Fly_ash</th>\n",
       "      <th>Water</th>\n",
       "      <th>Super_plasticizer</th>\n",
       "      <th>Coarse_aggregate</th>\n",
       "      <th>Fine_aggregate</th>\n",
       "      <th>Age</th>\n",
       "      <th>Comp_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1040.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>79.986111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1055.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>61.887366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>270</td>\n",
       "      <td>40.269535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>365</td>\n",
       "      <td>41.052780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>198.6</td>\n",
       "      <td>132.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978.4</td>\n",
       "      <td>825.5</td>\n",
       "      <td>360</td>\n",
       "      <td>44.296075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cement  BF_slag  Fly_ash  Water  Super_plasticizer  Coarse_aggregate  \\\n",
       "0   540.0      0.0      0.0  162.0                2.5            1040.0   \n",
       "1   540.0      0.0      0.0  162.0                2.5            1055.0   \n",
       "2   332.5    142.5      0.0  228.0                0.0             932.0   \n",
       "3   332.5    142.5      0.0  228.0                0.0             932.0   \n",
       "4   198.6    132.4      0.0  192.0                0.0             978.4   \n",
       "\n",
       "   Fine_aggregate  Age   Comp_str  \n",
       "0           676.0   28  79.986111  \n",
       "1           676.0   28  61.887366  \n",
       "2           594.0  270  40.269535  \n",
       "3           594.0  365  41.052780  \n",
       "4           825.5  360  44.296075  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete = pd.read_excel('../../../data/Concrete_Data.xlsx')\n",
    "concrete.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1030, 9)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_inputs = concrete.drop(\"Comp_str\", axis = 1).values\n",
    "train_targets = concrete[\"Comp_str\"].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Scaling "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_inputs =  preprocessing.scale(train_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1030, 8)\n",
      "(1030,)\n"
     ]
    }
   ],
   "source": [
    "print(train_inputs.shape)\n",
    "print(train_targets.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_cols = train_inputs.shape[1]\n",
    "input_shape = (n_cols, )\n",
    "\n",
    "# Creates a model given an activation and learning rate\n",
    "def create_model(learning_rate=0.01, activation='relu'):\n",
    "  \n",
    "    # Create an Adam optimizer with the given learning rate\n",
    "    opt = Adam(lr=learning_rate)\n",
    "  \n",
    "    # Create your binary classification model  \n",
    "    model = Sequential()\n",
    "    model.add(Dense(128, activation = activation,\n",
    "                         input_shape = input_shape,\n",
    "                         activity_regularizer=regularizers.l2(1e-5)))\n",
    "    model.add(Dropout(0.50))\n",
    "    model.add(Dense(128, activation = activation, \n",
    "                         activity_regularizer=regularizers.l2(1e-5)))\n",
    "    model.add(Dropout(0.50))\n",
    "    model.add(Dense(1, activation = activation))\n",
    "\n",
    "# Compile the model\n",
    "    model.compile(optimizer = opt,\n",
    "                  loss = \"mean_absolute_error\",\n",
    "                  metrics=['mse', \"mape\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-02-19 10:18:42.062387: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-02-19 10:18:42.063036: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:42.063215: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:42.063330: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:42.452604: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:42.452776: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:42.452899: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-02-19 10:18:42.453001: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 10976 MB memory:  -> device: 0, name: NVIDIA TITAN Xp, pci bus id: 0000:01:00.0, compute capability: 6.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 13 calls to <function Model.make_test_function.<locals>.test_function at 0x7f29686d69d0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x7f2968207ca0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    }
   ],
   "source": [
    "# Create a Classifier\n",
    "model = KerasRegressor(build_fn = create_model,\n",
    "                       verbose = 0)\n",
    "\n",
    "# Define the parameters to try out\n",
    "params = {'activation': [\"relu\", \"tanh\"],\n",
    "          'batch_size': [16, 32, 64], \n",
    "          'epochs': [50, 100],\n",
    "          'learning_rate': [0.01, 0.001, 0.0001]}\n",
    "\n",
    "# Create a randomize search cv object passing in the parameters to try\n",
    "random_search = RandomizedSearchCV(model,\n",
    "                                   param_distributions = params,\n",
    "                                   cv = KFold(10))\n",
    "\n",
    "random_search_results = random_search.fit(train_inputs, train_targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  -6.493083047866821 and Best Params:  {'learning_rate': 0.01, 'epochs': 100, 'batch_size': 32, 'activation': 'relu'}\n"
     ]
    }
   ],
   "source": [
    "# Print best score and parameters\n",
    "print(\"Best Score: \", random_search_results.best_score_, \"and Best Params: \", random_search_results.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fitting model with best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_cols = train_inputs.shape[1]\n",
    "input_shape = (n_cols, )\n",
    "\n",
    "# Creates a model given an activation and learning rate\n",
    "def create_model(learning_rate=0.001, activation='relu'):\n",
    "  \n",
    "    # Create an Adam optimizer with the given learning rate\n",
    "    opt = Adam(lr=learning_rate)\n",
    "  \n",
    "    # Create your binary classification model  \n",
    "    model = Sequential()\n",
    "    model.add(Dense(128, activation = activation,\n",
    "                         input_shape = input_shape,\n",
    "                         activity_regularizer=regularizers.l2(1e-5)))\n",
    "    model.add(Dropout(0.50))\n",
    "    model.add(Dense(128, activation = activation, \n",
    "                         activity_regularizer=regularizers.l2(1e-5)))\n",
    "    model.add(Dropout(0.50))\n",
    "    model.add(Dense(1, activation = activation))\n",
    "\n",
    "# Compile the model\n",
    "    model.compile(optimizer = opt,\n",
    "                  loss = \"mean_absolute_error\",\n",
    "                  metrics= ['mse', \"mape\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean accuracy was: -6.3261659145355225\n",
      "With a standard deviation of: 1.809826443693863\n"
     ]
    }
   ],
   "source": [
    "# Create a KerasRegressor\n",
    "model = KerasRegressor(build_fn = create_model,\n",
    "                       epochs = 100, \n",
    "                       batch_size = 16,\n",
    "                       verbose = 0)\n",
    "\n",
    "# Calculate the accuracy score for each fold\n",
    "kfolds = cross_val_score(model,\n",
    "                         train_inputs,\n",
    "                         train_targets,\n",
    "                         cv = 10)\n",
    "\n",
    "# Print the mean accuracy\n",
    "print('The mean accuracy was:', kfolds.mean())\n",
    "\n",
    "# Print the accuracy standard deviation\n",
    "print('With a standard deviation of:', kfolds.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2nd - Titanic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 12 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   PassengerId  891 non-null    int64  \n",
      " 1   Survived     891 non-null    int64  \n",
      " 2   Pclass       891 non-null    int64  \n",
      " 3   Name         891 non-null    object \n",
      " 4   Sex          891 non-null    object \n",
      " 5   Age          714 non-null    float64\n",
      " 6   SibSp        891 non-null    int64  \n",
      " 7   Parch        891 non-null    int64  \n",
      " 8   Ticket       891 non-null    object \n",
      " 9   Fare         891 non-null    float64\n",
      " 10  Cabin        204 non-null    object \n",
      " 11  Embarked     889 non-null    object \n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 83.7+ KB\n",
      "__________________________________________________\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 418 entries, 0 to 417\n",
      "Data columns (total 11 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   PassengerId  418 non-null    int64  \n",
      " 1   Pclass       418 non-null    int64  \n",
      " 2   Name         418 non-null    object \n",
      " 3   Sex          418 non-null    object \n",
      " 4   Age          332 non-null    float64\n",
      " 5   SibSp        418 non-null    int64  \n",
      " 6   Parch        418 non-null    int64  \n",
      " 7   Ticket       418 non-null    object \n",
      " 8   Fare         417 non-null    float64\n",
      " 9   Cabin        91 non-null     object \n",
      " 10  Embarked     418 non-null    object \n",
      "dtypes: float64(2), int64(4), object(5)\n",
      "memory usage: 36.0+ KB\n"
     ]
    }
   ],
   "source": [
    "train_data = pd.read_csv(\"../../../data/kaggle_titanic_train.csv\")\n",
    "test_data = pd.read_csv(\"../../../data/kaggle_titanic_test.csv\")\n",
    "\n",
    "train_data.info()\n",
    "print('_'*50)\n",
    "test_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_uuid": "23de2def20bde7868eb192a2c5edb9769bed11d9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "6aa1d7bbd96a25ce685a1d8fe7fa9451e7b069d4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>714.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.383838</td>\n",
       "      <td>2.308642</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.523008</td>\n",
       "      <td>0.381594</td>\n",
       "      <td>32.204208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>257.353842</td>\n",
       "      <td>0.486592</td>\n",
       "      <td>0.836071</td>\n",
       "      <td>14.526497</td>\n",
       "      <td>1.102743</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>49.693429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>223.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.125000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>668.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>512.329200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
       "count   891.000000  891.000000  891.000000  714.000000  891.000000   \n",
       "mean    446.000000    0.383838    2.308642   29.699118    0.523008   \n",
       "std     257.353842    0.486592    0.836071   14.526497    1.102743   \n",
       "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
       "25%     223.500000    0.000000    2.000000   20.125000    0.000000   \n",
       "50%     446.000000    0.000000    3.000000   28.000000    0.000000   \n",
       "75%     668.500000    1.000000    3.000000   38.000000    1.000000   \n",
       "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
       "\n",
       "            Parch        Fare  \n",
       "count  891.000000  891.000000  \n",
       "mean     0.381594   32.204208  \n",
       "std      0.806057   49.693429  \n",
       "min      0.000000    0.000000  \n",
       "25%      0.000000    7.910400  \n",
       "50%      0.000000   14.454200  \n",
       "75%      0.000000   31.000000  \n",
       "max      6.000000  512.329200  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "93aa50436d06d1fdece8970d469b0b329ef45d8c"
   },
   "source": [
    "#### Drop the columns if the missing values are more than 20% of the data. Drop missing rows otherwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_uuid": "9b18e532419264e9ca433a984e3974a0065ba3a5"
   },
   "outputs": [],
   "source": [
    "for col in train_data.columns:\n",
    "    if len(train_data[col].dropna()) <= (0.7 * len(train_data)):\n",
    "        train_data.drop(columns=[col], inplace=True)\n",
    "    else:\n",
    "        train_data.dropna(axis=0, subset=[col],inplace=True)\n",
    "\n",
    "for col in test_data.columns:\n",
    "    if len(test_data[col].dropna()) <= (0.7 * len(test_data)):\n",
    "        test_data.drop(columns=[col], inplace=True)\n",
    "    else:\n",
    "        test_data[col].fillna(value=test_data[col].mode()[0] ,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_uuid": "0c09d970d277431fa78e24ec009ea51bdc64fa41"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 712 entries, 0 to 890\n",
      "Data columns (total 11 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   PassengerId  712 non-null    int64  \n",
      " 1   Survived     712 non-null    int64  \n",
      " 2   Pclass       712 non-null    int64  \n",
      " 3   Name         712 non-null    object \n",
      " 4   Sex          712 non-null    object \n",
      " 5   Age          712 non-null    float64\n",
      " 6   SibSp        712 non-null    int64  \n",
      " 7   Parch        712 non-null    int64  \n",
      " 8   Ticket       712 non-null    object \n",
      " 9   Fare         712 non-null    float64\n",
      " 10  Embarked     712 non-null    object \n",
      "dtypes: float64(2), int64(5), object(4)\n",
      "memory usage: 66.8+ KB\n",
      "__________________________________________________\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 418 entries, 0 to 417\n",
      "Data columns (total 10 columns):\n",
      " #   Column       Non-Null Count  Dtype  \n",
      "---  ------       --------------  -----  \n",
      " 0   PassengerId  418 non-null    int64  \n",
      " 1   Pclass       418 non-null    int64  \n",
      " 2   Name         418 non-null    object \n",
      " 3   Sex          418 non-null    object \n",
      " 4   Age          418 non-null    float64\n",
      " 5   SibSp        418 non-null    int64  \n",
      " 6   Parch        418 non-null    int64  \n",
      " 7   Ticket       418 non-null    object \n",
      " 8   Fare         418 non-null    float64\n",
      " 9   Embarked     418 non-null    object \n",
      "dtypes: float64(2), int64(4), object(4)\n",
      "memory usage: 32.8+ KB\n"
     ]
    }
   ],
   "source": [
    "train_data.info()\n",
    "print('_'*50)\n",
    "test_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "6786728f751be8769471c80f5daadd20839ed7ff"
   },
   "source": [
    "No need to convert the data types from categorical to numerical because we want tensorflow to handle this kind of data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "af22d7039852eebe20955e0de24bc949093dbcfb"
   },
   "source": [
    "#### Define Feature columns for tensorflow  \n",
    "Examples of each column type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "_uuid": "f4994ae2c85b8c73ec22b312a12e15e369572280"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAHWCAYAAABQYwI2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACBF0lEQVR4nOzdd3hU1dbA4d8kpCekQwi9bum9d6SIDUVBEHtB7ocidmwgNrj2AlfsF1GUIgqoV0F6771saughvYe0me+PGVJIIAOZatbrM4855+yZszbT1qy9zzkGk8mEEEIIIYQ78HB2AEIIIYQQ1pLERQghhBBuQxIXIYQQQrgNSVyEEEII4TYkcRFCCCGE25DERQghhBBuo4qzAxBCCCGE+1FKfQPcDMRprVuUsd0AfAzcCGQBD2itt1d0v1JxEUIIIcS1+C9wwxW2DwYaW26jgc9ssVNJXIQQQghx1bTWq4GkKzQZAnyntTZprTcCIUqpGhXdryQuQgghhLCHmsCpYsunLesqxBFzXOSaAkIIISobgyN35ldnpM2/a+v4bX8M8xDPRV9orb+w9X6ulkMm5/rVGemI3bic7JM/AjBp+99OjsQ5JrfrD8CMA0ucHIlzjGk6EKicz//F537usT+dHIlzDG9gHvb/cO9SJ0fiHE+1GADA1F2Vs/8TWg9wdgg2YUlSKpKonAFqF1uuZVlXIXJUkRBCCOHmDAaXnPmxCHhcKfUT0BlI1Vqfq+iDSuIihBBCuDmDE6asKqV+BPoAEUqp08AkwAtAaz0D+APzodBHMB8O/aAt9nvFxEUp9SlXmKOitR5niyCEEEII4V601lecB6K1NgFjbb3f8iouWy3/7w40A+ZYlocB+20djBBCCCGunosOFdnFFRMXrfVMAKXUv4AeWut8y/IMYI39wxNCCCGEKGLtHJdQoCpFJ5oJtKwTQgghhJNJxaW0qcAOpdQKzMem9wJes1dQQgghhLCeweDQ08Y4lVUpmtb6W8yHMv0CLAC6XhxGEkIIIYRwlPKOKmp3yaqLp+6NVkpF2+Iqj0IIIYSoKBkquuj9K2wzAf1sGIsQQgghxBWVd1RRX6WUB+ahoXUOikkIIYQQV6EyTc4tt6daayMwzQGxCCGEEEJckbUp2jKl1B1KqcozbVkIIYRwEwaDh81vrsraw6EfA54GCpRS2ZgPiTZpravaLTIhhBBCWMUZ1ypyFqsSF611kL0DEUIIIYQoj1WJi2WIaBRQX2v9hlKqNlBDa73ZrtEJIYQQolyuPLRja9b29D9AV+Buy3IGMN0uEQkhhBBCXIa1iUtnrfVY4AKA1joZ8LZbVEIIIYSwmkzOLS1PKeWJ+aRzKKUiAaPdohJCCCGE1Vw50bA1a3v6CebrFFVTSr0FrAXetltUQgghhBBlsPaooh+UUtuA6zEfCn2b1vqAXSMTQgghhFUMVJ7TrFlVcVFKhQFxwI/AbOC8UsrLnoEJIYQQQlzK2jku24HaQDLmiksIEKuUOg88qrXeZp/whBBCCFEemeNS2lLgRq11hNY6HBgM/Ab8H+ZDpYUQQgjhJHJUUWldtNaPXlzQWi9RSr2ntX5MKeVjp9iuaMa7jzH4+rbEJ6bRYcDzZbZ5f/L9DOrbhqzsXEY/8xk798YAMOrOXkx44jYApn76Kz/MX+2gqG3n3M59bP9uPiajkQZ9u9NsyMAS2wvy8tj4n+9IPn4S78AAuj35MIGR4SQeiWHLV7PNjUzQ4s4bqdWxjeM7UEEmk4mVX/3M8W378PLxZuC4e6jesHapduePnOSvT74nPzeP+u2b0+eROzAYisaCt/26jNX//ZUx303Br2qgI7tQIfL8m/hjxgIObdmPl48XQ58ZRXSj0s//mcOnWPDBD+Tn5NGkYzNuHDMUg8HAuaOnWfTpXPLz8vHw9OCWscOopeo6oSfXxmQyse6b+Zzcvo8q3t70feJeIhuU7n/80ZOsmDaL/Nw86rRrTveH7sRgMHB0/Xa2zvmD5DPnGTr1Wao1cp++n965n03fml/7Ta7vRqvbSr/2V0+bReKxk/gEBdBn/EMEVQvnQnoGKz74moQjJ2jUpwtdHx7upB6IirI2pTqnlHpBKVXXcnse8zwXT5x0WPSseasYct/Uy24f1LcNDetF0aLXUzw+4Us+eethAEKDA3h5/FB63foqPW99lZfHDyUkOMBRYduE0Whk67dz6f3CWAa/9yon128l9fS5Em2OrdiAd4A/N380GXVjP3bN/hWA4NrRDHzrBW6Y+hK9J4xly1c/YiwocEIvKiZm235SzsXx4GcT6f9/I1g+Y06Z7ZZ9PocBY0fy4GcTSTkXR8z2/YXb0uOTObHzIEGRoY4K2ybk+YfDW/aTeDae8V+/wpBxI1g8bV6Z7RZPm8tt40Yw/utXSDwbz+Gt5mMK/vp6EX1H3cDY6c9z/T2D+evrRY4Mv8JObt9P6rl4Rk6bRO9/jWTNFz+V2W71F3Po/a+7GTltEqnn4jm1w/z6D6sTzaDnH6VGs4aODLvCjEYjG7+ey8CX/o/bP3yFY+u2kXLJa//Q8g34BPhx56ev0fymvmz9YSEAnl5etLvrZjree7szQre7ylRxsTayu4FawK+WWx3LOk/AKWnrus0HSUrJuOz2mwe2Z/bPawDYvOMIwVX9iaoWwoDerVm2Zg/JqZmkpGaybM0eBvZu7aiwbSLpSAxBUZEEVo/As0oV6nRtz5mtu0u0ObNtN/V7dQagdue2nN+rMZlMVPHxxsPTEzD/MsFNZ6If3byHpn06YTAYqKHqk5OZTUZSaok2GUmp5GZdoIaqj8FgoGmfThzdtKdw+8pvFtDz/iFuNxtfnn84sHEvba7viMFgoHbTemRnZJN+yfOfnpRKTtYFajeth8FgoM31HTmwwfz8GwwGcrIuAHAh6wJB4e51vdiYLbtp0tv8+q/exPz6z0wu2f/M5FTysi5QvYn59d+kdyeObza/TkJrRRFSs7ozQq+QhCMxBEVFEGR57Tfo1o6TW0q+9k9u3U2jPubXfr0ubTlnee17+fpQ/bqGeHrLcSXuztrDoROAJy6z+YjtwrGd6KgwTp9LLFw+E5tEdFQY0VGhnD6bVLT+XBLRUe71izs7OQX/8KKY/cJDSDoSU7JNUlEbD09PvPz9yE3PxKdqIIlHjrNpxvdkJSTRZez9hV9k7iQjKYWgiKJ/g8DwEDKSUgkMCy7WJpXA8JBL2qQAcHTTbgLDg4msX8tRIduMPP+QlphCcERI4XJwRDBpCakEFXv+0xJSqVqsTdWIENISUwAY/NjtfPfKZ/z51UJMJhOj3x/vmMBtJDMphcBLXv+ZiSkEhBb1PzMxhYBLXv+Zlte/u8pKSiWg2GvfPzyU+MMxl23j4emJt78fOemZ+LrRUPC1cd0Kia1Ze5HFJsCzQL3i99Fa97NPWMKewhvV58b3XiX1TCybPvuOGq2bV6pfIXk5uWyev4Shr411dihOUdmff4Atv69j8Ojbad6jDXtW7+CXj37kwSmV8/Ug/hlceWjH1qydnDsPmAF8BbjFgPjZ2CRq1QgvXK4ZFcbZ2CTOxibTs2vTovU1wlizwb3OpecXGkJWYnLhcnZiCn6hISXbhJnb+IeHYiwoIC8rG++gknN5gmtGUcXHh9RTZwlr6PqT83b+sZq9S9YDUL1xHdITiv4NMhJTSlRbAALDgsmw/MIuahNC6rkEUuMS+X68eY5UemIKPzz9DiPffZaAUNcfMqisz/+mxWvY+ucGAGo2qUNqQkrhttSEVKpGlHz+q0YEk1asTVpCClUtFYgdf2/mxjFDAWjRsw0LP/rRrrHbwt7/reLA3+bXf2SjumRc8vovXl0BCLBUYUq0CSvZxt34hwWTWey1n5WYTMAl7/uLbQIsr/3crGx8gtxrHqO4MmsTl3yt9Wd2jcTGfl+6nTH3D2TuovV0atuItPQsYuNSWLpqF5Ofv6twQm7/nq2YOLXsiW2uKqxhXdJj48iIS8AvLISTG7bR9fEHSrSp2b4lx1dvIqJJA05t2kH15k0wGAxkxCXgHx6Kh6cnmfGJpJ09T0BkeNk7cjFtbuxFmxt7AXBs6152/bEa1bM9sYdi8A7wLTNx8fb35Zw+TlSTehxYuZk2N/Ymol40Y2ZOKWz39aOTuPv959zmqKLK+vx3vqUnnW/pCYDevI9Ni9fQsnc7Th88gW+Ab4lhIoCgsGB8/H05dSCGWtfVZeeyLXSx3D8oPJiYPUeo36oxx3YeIrxmpMP7c7VaDO5Ni8G9ATixbS97/7eaRj3aE3c4Bm9/vxLDRAABocF4+fty/tBxqjWux6FVmwvv764iGtYl7Vw86XEJ+IeFcGz9dnqPe6BEmzrtW3Jk5SaqNWlAzMYd1LC89v/ppOJS2mKl1P9hvl5RzsWVWuuky9/FvmZ++gQ9uzYlIjSII5um8cYH8/HyMnfnq+//5s/lOxjUtw371nxEVnYOjz37OQDJqZlM+eQX1i5+E4C3P15Acmqms7pxTTw8PWn/wHBWTZmO0WikQZ+uBNeOZs+83wirX4eaHVrRoE83Nv5nJr+Nn2Q+HPaJhwCI10c5sHAJHlU8MRg86PDQXfi4yRd2cfXbNydm236+HfM6VXy8GDjunsJt34+fyj0fTQCg32N3seST78nPyaNe+6bUa9/MWSHbjDz/0KRjMw5t2c+HD72Bl683Q5+6u3Db9LHvMHa6+RQJt4wdxoIPfiDPcjh0447m5/+2cXfxx+cLKCgw4uXtxa3jRjilH9eqTrvmnNy+jx/HTqaKjxd9xha9/uc9M4Vh778IQM9Hh7Ni2vcU5OZRu20z6rQz9//4pl2s/Woe2WkZ/O/tGYTXq8nNEx93Sl+uhoenJ10eGs6St6ZjMppo3LcLobVrsH3Ob0Q0rEOdDq1o3K8ba6Z9x/wnXsMnMIA+4x8svP+8sRPJzbqAMT+fk1t2M+iVsYTUquHEHolrYTCZTOU2UkodL2O1SWvdwIp9mPzqjLzqwP4Jsk+ay8+Ttv/t5EicY3K7/gDMOLDEyZE4x5im5vNLVMbn/+JzP/fYn06OxDmGN7gBgA/3LnVyJM7xVIsBAEzdVTn7P6H1AHDwIXs1W0wq/8v8Kp3ZO9klS1XWHlVU396BCCGEEEKUx9qjivyBp4E6WuvRSqnGgNJa/2bX6IQQQghRrso0x8Xann4L5ALdLMtngDftEpEQQgghrorBYLD5zVVZm7g01Fq/A+QBaK2zcNdTbgohhBDCbVl7VFGuUsoPMAEopRpS7OgiIYQQQjhPZRoqsjZxmQT8CdRWSv0AdAcesFdQQgghhBBlsfaooqVKqe1AF8xDRE9arl8khBBCCCczVKJrFVnVU6VUd+CC1vp3IAR4SSnl+ucIF0IIISoBg8HD5jdXZW1knwFZSqnWmA+LPgp8Z7eohBBCCCHKYG3ikq+1NgFDgOla6+lAkP3CEkIIIYS1KlPFxdrJuelKqReBe4BeSikPwMt+YQkhhBBClGZtSnUX5sOfH9ZaxwK1gHftFpUQQgghrGbAw+Y3V2V1xQX4WGtdoJRqAlwH/Gi/sIQQQghhNRce2rE1a3u6GvBRStUElgD3Av+1V1BCCCGEEGWxNnExWE7zPxT4j9Z6GNDCfmEJIYQQwlqVaXKu1YmLUqorMAr4/SrvK4QQQghhE9bOcXkSeBH4RWu9TynVAFhhv7CEEEIIYS1XvpqzrVl7yv/VmOe5XFw+BoyzV1BCCCGEcH1KqRuAjwFP4Cut9dRLttcBZmI+674nMEFr/UdF9mlV4qKUigSeB5oDvhfXa637VWTnQgghhKg4Zxy+rJTyBKYDA4DTwBal1CKt9f5izV4B5mqtP1NKNQP+AOpVZL/W9vQH4CBQH5gMxABbKrJjIYQQQtiGkybndgKOaK2Paa1zgZ8wn2G/OBNQ1fJ3MHC2on21NnEJ11p/DeRprVdprR8CpNoihBBCVF41gVPFlk9b1hX3GnCPUuo05mrLExXdqbWTc/Ms/z+nlLoJc8YUVtGdCyGEEMIG7DA5Vyk1GhhdbNUXWusvrvJhRgL/1Vq/bzk6eZZSqoXW2nitcVmbuLyplAoGngE+xVz2eepadyqEEEII12ZJUq6UqJwBahdbrmVZV9zDwA2Wx9uglPIFIoC4a43riomLZQdjgEaYyz9fa637XuvOhBBCCGEHzjmz2hagsVKqPuaEZQRw9yVtTgLXA/9VSjXFfIBPfEV2Wl5XZwIdgD3AYOD9iuxMCCGEEHZgMNj+Vg6tdT7wOPAXcADz0UP7lFKvK6VutTR7BnhUKbUL8zUOH9BamyrS1fKGippprVsCKKW+BjZXZGdCCCGE+OewnJPlj0vWTSz2936guy33aTCZLp/4KKW2a63bXW7ZShXKrIQQQgg35NBT2TbpNsPm37WH1o9xydPxlldxaa2USrP8bQD8LMsGwKS1rnr5uwohhBBC2NYVExettactdjJp+9+2eBi3M7ldfwD86ox0ciTOkX3yRwCSchY5ORLnCPMxD/HOPfankyNxvOENbgDg431LnByJczzZfCAAj2+onJd0m9bVfAxHtwVrnRyJc6wf2sPxO61Elz229nBoIYQQQrgoUyW6yGIlytGEEEII4e6k4iKEEEK4u8pTcJGKixBCCCHch1RchBBCCHfnUXlKLlJxEUIIIYTbkIqLEEII4e4q0VFFkrgIIYQQ7q7y5C0yVCSEEEII9yEVFyGEEMLdVaLJuVdMXJRS6VzhIolyrSIhhBBCOFJ51yoKAlBKvQGcA2ZhHkkbBdSwe3RCCCGEKJ9Mzi3lVq1162LLnymldgET7RCTEEIIIa5G5clbrE5cMpVSo4CfMA8djQQy7RaVEEIIIUQZrD2q6G5gOHDechtmWSeEEEIIZ/Mw2P7moqyquGitY4Ah9g1FCCGEEOLKrKq4KKWaKKWWKaX2WpZbKaVesW9oQgghhLCKwQ43F2XtUNGXwItAHoDWejcwwl5BCSGEEMJ6JoPB5jdXZW3i4q+13nzJunxbByOEEEIIcSXWHlWUoJRqiOVkdEqpOzGf10UIIYQQzubCk2ltzdrEZSzwBXCdUuoMcBzzSeiEEEIIIRzG2sTlhNa6v1IqAPDQWqfbMyghhBBCXIXKU3Cxeo7LcaXUF0AXIMOO8QghhBBCXJa1ict1wN+Yh4yOK6WmKaV62C8sIYQQQljNYLD9zUVZewK6LGAuMFcpFQp8DKwCPO0YW7nO7dzH9u/mYzIaadC3O82GDCyxvSAvj43/+Y7k4yfxDgyg25MPExgZTuKRGLZ8NdvcyAQt7ryRWh3bOL4DFTDj3ccYfH1b4hPT6DDg+TLbvD/5fgb1bUNWdi6jn/mMnXtjABh1Zy8mPHEbAFM//ZUf5q92UNS2YzKZ+PDfC1m/5iC+vl68+sZdqGa1SrUbP+ZLEhPSKSgw0rpdfZ596XY8Pc35+rzZa5n/03o8PT3o1vM6Hn/6Zkd345qZTCb+mLGAQ1v24+XjxdBnRhHdqHapdmcOn2LBBz+Qn5NHk47NuHHMUAwGA+eOnmbRp3PJz8vHw9ODW8YOo5aq64SeXBuTycTar3/mxPZ9VPHx5vrH7yGyYen+xx09yfJPvyc/N4+67ZrT4+E7MBgMrJ/5KzFb9+BRpQrB1SPo98QofAL8ndCTa5Owex969lxMRiM1e3Wn/s03lNierA+jZ88l49QZWv7rYap3bF+47ezaDRxb/AcADW65kegeXR0ae0V1rh7C+FYN8DQYWBxznlmHTpfYPqJRNLfUi6LAZCIlJ4+3tx0mNjuHdhHBjGtVv7Bd3SB/Jm0+yOpzSY7ugn1Uosm51lZcUEr1Vkr9B9gG+GK+BIDTGI1Gtn47l94vjGXwe69ycv1WUk+XPNDp2IoNeAf4c/NHk1E39mPX7F8BCK4dzcC3XuCGqS/Re8JYtnz1I8aCAif04trNmreKIfdNvez2QX3b0LBeFC16PcXjE77kk7ceBiA0OICXxw+l162v0vPWV3l5/FBCggMcFbbNbFh7kFMnEpj32wtMmHgn77y5oMx2b713L7PmP80PC54hJSmD5Ut2A7Bt8xFWr9jHrPlPM/uXZ7n7/j4OjL7iDm/ZT+LZeMZ//QpDxo1g8bR5ZbZbPG0ut40bwfivXyHxbDyHtx4A4K+vF9F31A2Mnf48198zmL++XuTI8Cvs5Pb9pJ6LY9T0ifQZM4JVX8wps93qz+fQ518jGTV9Iqnn4ji5Yz8AtVorRnz0EiM+fJGQ6Gps/3mpI8OvEJPRyMFZP9L26cfp9vYkYjdtIePM2RJtfMNCaf7I/UR16VhifV5GJscW/k7nVyfQeeIEji38nbxM97nsnAfwbOuGPLNuH3cv3U7/WpHUC/Ir0eZQSiYPrdjJfct2sOJMAv/Xsh4A2xNSeWD5Th5YvpMn1uwlp6CATXEpDu+DqDhrz5wbA4wH1gAttdbDtdY/2zGuciUdiSEoKpLA6hF4VqlCna7tObN1d4k2Z7btpn6vzgDU7tyW83s1JpOJKj7eeHiai0UFeXm446ymdZsPkpRy+elGNw9sz+yf1wCweccRgqv6E1UthAG9W7NszR6SUzNJSc1k2Zo9DOzd+rKP46pWr9jH4FvaYzAYaNG6LhnpF0iITyvVLiDQF4CCfCN5eQWF1c8Fczdw78N98fY2Fx3DwgMdFrstHNi4lzbXd8RgMFC7aT2yM7JJT0ot0SY9KZWcrAvUbloPg8FAm+s7cmDDHgAMBgM5WRcAuJB1gaDwqg7vQ0Uc37wH1acTBoOBKFWf3MxsMi/pf2ZSKrnZF4hS9TEYDKg+nTi+ydz/Om2aFn4GVG9Sj4zEFEd34ZqlHovBv3o1/KtF4lGlClGdOxK/o+Rnn19kBEG1a5Uq9yfs3U9Y86Z4BQbgFRBAWPOmJOzZ78jwK6RZWBCnMy9wNiuHfJOJv0/H07NGeIk22xNSySkwArAvKZ1qfj6lHqdfzXA2xCYXtvtHqERnzrX2qKJWWuvS3wpOlJ2cgn94aOGyX3gISUdiSrZJKmrj4emJl78fuemZ+FQNJPHIcTbN+J6shCS6jL2/8EPsnyI6KozT5xILl8/EJhEdFUZ0VCinzxaVRs+cSyI6KrSsh3Bp8XFpVI8KKVyOrB5MfFwqEZGlv4DHj/mS/XtO0aWHou+AVgCcOhHPrm3H+fyTP/H28eKJZ26mWYvSQw2uKi0xheCIkMLl4Ihg0hJSCQoLLmqTkErVYm2qRoSQZvmCHvzY7Xz3ymf8+dVCTCYTo98f75jAbSQzKYXAiKLXbUB4CJlJqQQU639mUiqB4SGXtEkp9VgHlm+kUfd29gzXpnKSk/EJK+q7T2gIaceOW31f32L39Q0NISc52eYx2kukrzfns3MKl+Ozc2gWFnTZ9jfXq87G2NL9618rkh+PnLFLjML+rpi4KKWe11q/A7yllDJdul1rPc5ukdlZeKP63Pjeq6SeiWXTZ99Ro3VzPL29nB2WsIOPZjxKTk4er02YzbbNR+jUtQkF+UbS0rL56ocn2L/3FK88O4uf//ciBheekGZLW35fx+DRt9O8Rxv2rN7BLx/9yINTxjo7LIfbOv8vPDw8aNKrg7NDETY2qHYk14UGMnb1nhLrw329aBAcwKbzKc4JzF4qyWcXlF9xOWD5/1Z7B3K1/EJDyEosyqSzE1PwCw0p2SbM3MY/PBRjQQF5Wdl4B5WczxFcM4oqPj6knjpLWEP3mZxYnrOxSdQqVkKtGRXG2dgkzsYm07Nr06L1NcJYs+FAWQ/hcub/tI5FP28CoGnz2pyPTSncFn8+lchqwZe5J/j4eNGzb3NWr9hHp65NiKweTJ/rW2AwGGjesg4eHgZSkjMJDXPdIaNNi9ew9c8NANRsUofUhJTCbakJqVSNKNn/qhHBpBVrk5aQQlVLBWLH35u5ccxQAFr0bMPCj360a+y2sOd/q9m/dD0A1RrVISOh6P2fmZhSotoCEBAWXGIIyNwmpHD54PKNnNi6l1snP+FWCatPaCg5SUV9z0lOwSfUuqqpT2goyQcPFS5fSE4h9LomNo/RXuIv5FK92NBPpJ8P8dm5pdp1iAzmflWbsWv2kGcs+Zv7+pqRrD6bSIGp1G9x9+ZGr+GKuuIcF631Ysufe7TWMy+9OSC+ywprWJf02Dgy4hIoyM/n5IZt1GzfskSbmu1bcny1+Yvu1KYdVG/eBIPBQEZcQuFk3Mz4RNLOnicgMrzUPtzZ70u3c/cdPQHo1LYRaelZxMalsHTVLvr3bEVIcAAhwQH079mKpat2OTla69w5ojvfzXua7+Y9Ta9+Lfjf4m2YTCb27jpBQJBvqWGirKycwnkv+fkFrF9zkLr1qwHQq18Ltm05CsDJmHjy8goICXXtScqdb+nJ2OnPM3b68zTt2pKdy7ZgMpk4dSAG3wDfEsNEAEFhwfj4+3LqQAwmk4mdy7bQtEsL87bwYGL2HAHg2M5DhNeMdHh/rlbLwb2464MJ3PXBBOp3aoVeuRmTyUSsPo63v2+ZiYu3ny+x+jgmkwm9cjP1O5k/I05u38+OX5dx44uj8fLxdkZ3rlnV+nXJOh9HdnwCxvx8YjdtIbJtK6vuG9GiGYl795OXmUleZiaJe/cT0aKZnSO2nQPJ6dQK9KOGvw9VDAb614pk7SVHBTUJDuCFto14fsN+knPySj1G/9oRLD0V76iQhR1YO8flfaVUFDAfmKO13mvHmKzi4elJ+weGs2rKdIxGIw36dCW4djR75v1GWP061OzQigZ9urHxPzP5bfwk8+HQTzwEQLw+yoGFS/Co4onB4EGHh+7Cp6rr/tIuy8xPn6Bn16ZEhAZxZNM03vhgPl5e5qfzq+//5s/lOxjUtw371nxEVnYOjz37OQDJqZlM+eQX1i5+E4C3P15Acqr7HFVwUbee17F+zQGG3TQVH19vXnmj6CC3+4Z9wHfznuZCdi7Pj/uW3Nx8TEYT7To14vZhXQC45faOvDVxLqNuf48qXlV49c0RbvWru0nHZhzasp8PH3oDL19vhj51d+G26WPfYex08yHyt4wdxoIPfiDPcjh0447mL6nbxt3FH58voKDAiJe3F7eOc6+Lvddt35yT2/fzw/+9ThUfL/o9fk/htjlPT+WuDyYA0Gv0XYWHQ9dp15Q67cz9X/3VPAry8lk0eTpgnqDbZ4x7/Bt4eHqi7rmL7e99gsloJLpnNwJrRnNkwSKq1q9LtbatST0Ww65PZ5CXmUXCzj0c/eU3ur09Ca/AABrceiObJpuPSGww5Ca8Al07YS+uwAQf7DzKh91b4GmA306c53h6Fo80rcPBlAzWnktibMv6+FXx5M3O1wFwPjuHFyxV5Sh/H6r7+bAjIfVKu3FPVh8j7P4MJivLZZbEZThwF1AVcwLzphV3NU3a/ve1R+jGJrfrD4BfnZFOjsQ5sk+ahx+SctzrUFtbCfO5FYC5x/50ciSON7yB+bwiH+9b4uRInOPJ5uZzSj2+YYWTI3GOaV37AtBtwVonR+Ic64f2AAcfl9No2Pc2H/s6Mu8el/w1Z3WOprWO1Vp/AowBdgIT7RWUEEIIIa6CnDm3JKVUU8yVljuARGAO8Iwd4xJCCCGEtVw3z7A5a+e4fAP8BAzSWp8tr7EQQgghhD2Um7gopTyB41rrjx0QjxBCCCGukkmuVVREa10A1FZKudcxg0IIIYT4x7F2qOg4sE4ptQgoPHZWa/2BXaISQgghhPVceDKtrVmbuBy13DyAy18YQgghhBDCjqxKXLTWk+0diBBCCCGuUeUpuFh9OPQKoKyLLPazeURCCCGEuDqVaHKutUNFzxb72xfz+VzybR+OEEIIIcTlWTtUtO2SVeuUUpvtEI8QQgghrpZMzi1JKRVWbNED6AAEX6a5EEIIISoBpdQNwMeAJ/CV1npqGW2GA69hnnKyS2t996Vtroa1Q0XbKJrjkg/EAA9XZMdCCCGEsBEnFFwsJ6idDgwATgNblFKLtNb7i7VpDLwIdNdaJyulqlV0v1dMXJRSHYFTWuv6luX7Mc9viQH2X+GuQgghhHAU50zO7QQc0VofA1BK/QQMoWR+8CgwXWudDKC1jqvoTss7c+7nQK4loF7AFGAmkAp8UdGdCyGEEMJt1QROFVs+bVlXXBOgiVJqnVJqo2VoqULKGyry1FonWf6+C/hCa/0z8LNSamdFdy6EEEIIG7BDxUUpNRoYXWzVF1rrqy1aVAEaA32AWsBqpVRLrXXKtcZVbuKilKqitc4HrqdkB6ydHyOEEEIIN2NJUq6UqJwBahdbrmVZV9xpYJPWOg84rpQ6hDmR2XKtcZWXfPwIrFJKJQDZwBoApVQjzMNFQgghhHAyk3OOht4CNFZK1cecsIwALj1i6FdgJPCtUioC89DRsYrs9IpzXLTWbwHPAP8FemitLx5Z5AE8UZEdCyGEEMJGPAy2v5XDMhrzOPAXcACYq7Xep5R6XSl1q6XZX0CiUmo/sAJ4TmudWJGuljvco7XeWMa6QxXZqRBCCCHcn9b6D+CPS9ZNLPa3CXjacrMJmacihBBCuLtKdObc8g6HFkIIIYRwGVJxEUIIIdxdJbo6tFRchBBCCOE2pOIihBBCuLtKVIaQxEUIIYRwd5Vocq7BZDKV36pi7L4DIYQQwsU4NJNo8PgvNv+uPTbtdpfMhqTiIoQQQri7SjQ51yGJy4wDSxyxG5czpulAAJJyFjk5EucI8zGfONGvzkgnR+Ic2Sd/BCA55zcnR+J4oT43A/D85uVOjsQ53unUD4DJO/52ciTOMaltf8tflfVcpU2cHcA/mlRchBBCCDdnqkRzXCRxEUIIIdxdJTqqqBJ1VQghhBDuTiouQgghhLurRJNzpeIihBBCCLchFRchhBDC3cnkXCGEEEK4DRkqEkIIIYRwPVJxEUIIIdxd5Sm4SMVFCCGEEO5DKi5CCCGEmzPJHBchhBBCCNcjFRchhBDC3VWiioskLkIIIYS7q0TncZGhIiGEEEK4Dam4CCGEEO6uEpUhrEpclFINgdNa6xylVB+gFfCd1jrFjrEJIYQQQpRgbY72M1CglGoEfAHUBmbbLSohhBBCWM9gsP3NRVmbuBi11vnA7cCnWuvngBr2C0sIIYQQVvMw2P7moqxNXPKUUiOB+4HfLOu87BOSEEIIIUTZrJ2c+yAwBnhLa31cKVUfmGW/sIQQQghhNReukNiaVYmL1no/MA5AKRUKBGmt/23PwIQQQgghLmXtUUUrgVst7bcBcUqpdVrrp+0YmxBCCCGsYHLhybS2Zu0cl2CtdRowFPNh0J2B/vYLSwghhBBW87DDzUVZG1oVpVQNYDhFk3OFEEIIIRzK2sm5rwN/AWu11luUUg2Aw/YLyzomk4mVX/3M8W378PLxZuC4e6jesHapduePnOSvT74nPzeP+u2b0+eROzAUK6tt+3UZq//7K2O+m4Jf1UBHduGamUwmPvz3QtavOYivrxevvnEXqlmtUu3Gj/mSxIR0CgqMtG5Xn2dfuh1PT3O+Om/2Wub/tB5PTw+69byOx5++2dHduGYz3n2Mwde3JT4xjQ4Dni+zzfuT72dQ3zZkZecy+pnP2Lk3BoBRd/ZiwhO3ATD101/5Yf5qB0VtOyaTiQ/+/Ssb1hzAx9ebV98YwXVlPv9fkJCQRkGBkTbtGvDsS0Px9PTg5ee+42RMPADp6dkEBfkxa94zju7GNTu/ex97Zs0Fo4k6fbrT5JZBJbYX5OWx/fOZpB4/iVdgAB0ffwT/yHBOrdvMkT+WFrZLO3WGPm+8SHDd0p8bruzszn1smzkfk9FIw37daT5kYIntBXl5bJj+HUnHT+ITGED3Jx8msFo453YfYOePCzHmF+BRxZO2o24nqoVyUi+ujclk4q23vmDVqm34+vowdeqTNG/eqFS7e+99kbi4ZHx9vQH45pvXCQ8PYcGCv3nnnW+pXj0cgHvuuYlhwwaVur/bqURDRdZOzp0HzCu2fAy4w15BWStm235SzsXx4GcTiT0Uw/IZcxj57rOl2i37fA4Dxo4kqkk9fn3jM2K276d+++YApMcnc2LnQYIiQx0dfoVsWHuQUycSmPfbC+zbfZJ33lzA17PHlWr31nv3EhDoi8lk4qWnv2P5kt0MGNyGbZuPsHrFPmbNfxpv7yokJWY4oRfXbta8VcyY+Rdfffh/ZW4f1LcNDetF0aLXU3Rq24hP3nqYXkNeJTQ4gJfHD6X7TS9jAtb//ha/L91GSmqmYztQQUXP/4uW5/9nvpn9ZKl2b713X+Hz/+LTM1m+ZBcDBrflrXfvK2zz8XuLCAz0dWT4FWIyGtk98ye6vTAOv7BQVk2cSlS7VlStWXRqqZOr1uMd4E//91/n9IYt7JvzCx0ff4Ta3TtRu3snwJy0bPpohtslLUajka3fzKXfy0/gFx7CXy+9Q632LQmuVdT/oys24B3oz60fTyZm/VZ2zv6VHuMfxicokN7PjcE/LISUU2dZ8fY0bv/sbSf25uqtXr2NmJizLFnyObt2aV577TPmzXu/zLbvvfcMLVs2LrX+xht7MnHiGHuHKuzEqqEipZSvUmqsUuo/SqlvLt7sHVx5jm7eQ9M+nTAYDNRQ9cnJzCYjKbVEm4ykVHKzLlBD1cdgMNC0TyeObtpTuH3lNwvoef8QDLhXtrp6xT4G39Ieg8FAi9Z1yUi/QEJ8Wql2AZYvpIJ8I3l5BYVJ+YK5G7j34b54e5tz17Bw96g0XbRu80GSUi6fbN08sD2zf14DwOYdRwiu6k9UtRAG9G7NsjV7SE7NJCU1k2Vr9jCwd2tHhW0zq1fs5cYSz3+2Vc//pS9zk8nEsr92MmBwW0eEbRPJR2MIqB5JQLVIPKpUoWaXDsRu21Wizbntu6jdowsA0Z3akbDvICaTqUSb0xu2ULNLB4fFbSuJR2IIjIoksHoEnlWqULdbe05v3V2izemtu6nfqzMAdTq35fw+jclkIqx+bfzDQgAIrlWDgtw8CvLyHN2FClm2bCO33dYPg8FAmzbXkZaWSVxckrPDcj45AV0ps4AoYBCwCqgFpNsrKGtlJKUQFFFUKQkMDykzcQkMD7mkTQoARzftJjA8mMj6pUvsri4+Lo3qUSGFy5HVg4mPSy2z7fgxX3Jjn8n4B/jQd0ArAE6diGfXtuM8fPcn/OvBz9i/95QjwnaY6KgwTp9LLFw+E5tEdFQY0VGhnD5b9CF35lwS0VHuVW0DiI9LpVqx57/aFZ7/J8d8zuA+kwgI8KHfgJJJ2s5txwgLD6JO3Uh7hmtTF5JT8Asres78wkK5kJxSsk1SCn7h5jYenp5U8fcjN6NkVe3Mpm3UcsPEJTsphYDwov77h4WQZflMK6uNh6cnXn5+5KSX7P+pTTsIrV8bTy/3Opfo+fOJREVFFC5HRYVz/nximW1feuljhgwZx/TpP5VIXJcsWc8ttzzBuHFTOHcu3u4xC9uyNnFppLV+FcjUWs8EbgI62y8s+8vLyWXz/CV0G3mTs0Oxu49mPMri5a+Sl5vPts1HAPMv8LS0bL764Qkef/omXnl2VqlfpOKf4eMZj/Hb8knk5uazdXPJqWlL/rfDraottpJ05Die3t5UrV3T2aE4Rcqps+ycvZBOj4x0dih28957z7J48TR++GEq27btY+HCFQD07duJ5cu/ZvHiT+nWrQ0vvPCRcwO1lUpUcbF2cu7FWmKKUqoFEAtUs09IV7bzj9XsXbIegOqN65CekFy4LSMxhcCw4BLtA8OCyUhMuaRNCKnnEkiNS+T78VMBSE9M4Yen32Hku88SEFrV/h25BvN/WseinzcB0LR5bc7HphRuiz+fSmS14MvcE3x8vOjZtzmrV+yjU9cmRFYPps/1LTAYDDRvWQcPDwMpyZmEhrnXkNHlnI1NolaN8MLlmlFhnI1N4mxsMj27Ni1aXyOMNRsOOCPEqzb/p7UsLPb8xxV7/uOseP579W3BmhX76NzVPBkzP7+Alcv2MPOnp+wat635hoaQnVT0vs9OSsY3NKRkm7AQshOT8QsLxVhQQH5WNt6BAYXbz2zcSq2u7ldtAfALCyEzsaj/WUkphcM/l7bxDzf3Py87G58gc/+zEpNZ8/6XdB17H0FR7lFp++GH35k79y8AWrZsTGxsQuG22NjEwom2xV1cFxjoz80392b37kPcdls/Qot9vg8bNpB33/2vfYN3FNfNM2zO2orLF5Yz5r4KLAL2A+/YLaoraHNjL+75aAL3fDSBhp1bcWDlZkwmE+f0cbwDfMtMXLz9fTmnj2MymTiwcjMNO7Ukol40Y2ZO4eEvJ/Pwl5MJCg9h1AfPu2zSAnDniO58N+9pvpv3NL36teB/i7dhMpnYu+sEAUG+RESWjD0rK6dw3kN+fgHr1xykbn1zvtmrXwu2bTkKwMmYePLyCggJDeCf4vel27n7jp4AdGrbiLT0LGLjUli6ahf9e7YiJDiAkOAA+vdsxdJVu8p5NNdw54gezJr3DLPmPUPvfi34o9jzH2jF879uzf7C5x9gy8bD1KtfrcSQkzsIaVCXzNg4MuMSMObnc2bjVqLatSrRJqptK06t3QjA2c3biWimCo8kNBmNnN28zS3ntwCEN6xLemwcGXEJFOTnc2L9Nmq2b1miTa32LTm+2pzknty0g+rNm2AwGMjNzGLlvz+jzd1DiFQNnRH+NRk16iYWLvyEhQs/oX//Lvz663JMJhM7dx4kKMifatXCSrTPzy8gyTJtIC8vn5Urt9C4cV2AEvNhli/fTMMyjkQVrs3ao4q+svy5Cmhgv3CuTv32zYnZtp9vx7xOFR8vBo67p3Db9+Oncs9HEwDo99hdLPnke/Jz8qjXvin12jdzVsg2063ndaxfc4BhN03Fx9ebV94YXrjtvmEf8N28p7mQncvz474lNzcfk9FEu06NuH2YecLiLbd35K2Jcxl1+3tU8arCq2+OKHGIuKub+ekT9OzalIjQII5smsYbH8zHy8v8cv7q+7/5c/kOBvVtw741H5GVncNjz34OQHJqJlM++YW1i98E4O2PF5DsZkcUAXTr2ZT1aw5w501T8PX14pU3RhRuu3fY+8ya9wzZ2bk8N+6bYs9/Q24f1rWw3dI/3XOYyMPTk1b3jWDDu59iMhqp06sbVWtFc+DnxYTUr0ONdq2p27s722f8l7+fmYhXoD8dxj5ceP9EfQS/sFACqrlHteFSHp6edHhwOCveno7JaKRB366E1I5m99zfCGtQh1odWtGwbzfWT5/Joicn4R0YQI9xDwFw6K9VpJ+PZ8/Pf7Dn5z8A6PfSE/gGBzmzS1eld+8OrFq1lQEDRuPn58PbbxcdTTdkyDgWLvyE3Nw8HnlkEnl5BRiNBXTt2obhw82HjM+atZjlyzfh6elJcHAQU6aUPhrPHZlceGjH1gxXmteglLriKf211h9YsQ/TjANLrjauf4QxTc1vlKScRU6OxDnCfG4FwK/OP3cc/UqyT/4IQHJO5TtnY6iP+ZxAz29e7uRInOOdTv0AmLzjbydH4hyT2l48sfohp8bhPE3AwYM3dT5YYfNJiief7uuS2VB5FRf3ScOFEEKIysqNKuYVdcXERWs92VGBCCGEEOIaVaKhImtPQDdTKRVSbDnUFU5AJ4QQQojKxdqjilpprVMuLmitkwH3m9UnhBBC/BMZ7HBzUdYmLh6Ww6EBUEqFYf05YIQQQgjxD6SUukEppZVSR5RSE67Q7g6llEkpVeHzEFibfLwPbFRKzbUsDwPequjOhRBCCFFxHtaWIWxIKeUJTAcGAKeBLUqpRVrr/Ze0CwKeBDbZYr9WdVVr/R1wO3DechuqtZ5liwCEEEIIUTEGg+1vVugEHNFaH9Na5wI/AUPKaPcG8G/ggi36esWKi1LKFxgDNAL2ADO01vm22LEQQggh3FpNoPgVek9zyXUMlVLtgNpa69+VUs/ZYqflDRXNxHydojXAYKApMN4WOxZCCCGEbdjjNC5KqdHA6GKrvtBaf3EV9/cAPgAesGVc5SUuzbTWLS0BfA1stuXOhRBCCOGaLEnKlRKVM0Dxiz3Vsqy7KAhoAaxUSgFEAYuUUrdqrbdea1zlJS4XrwqN1jrfsmMhhBBCuBAnXWtuC9BYKVUfc8IyArj74katdSoQcXFZKbUSeLYiSQuUn7i0VkqlWf42AH6WZQNg0lq77qWUhRBCiErCGXmLpaDxOPAX4Al8o7Xep5R6HdiqtbbLhfrKO+W/pz12KoQQQgj3p7X+A/jjknUTL9O2jy32KSeRE0IIIdxcJbrGotVnzhVCCCGEcDqpuAghhBBuzlCJyhCVqKtCCCGEcHdScRFCCCHcXGWa4yKJixBCCOHmPCpR4iJDRUIIIYRwG1JxEUIIIdxcZRoqkoqLEEIIIdyGVFyEEEIIN1eZKi6SuAghhBBuzkkXWXQKGSoSQgghhNswmEwme+/D7jsQQgghXIxDSyAtv1tj8+/aPff1dMkyjlRchBBCCOE2HDLHZdL2vx2xG5czuV1/AOYe+9PJkTjH8AY3AJCc85uTI3GOUJ+bAfCrM9LJkThe9skfATidudjJkThHrYBbANia8LuTI3GODhE3AfD4hhVOjsQ5pnXt6/B9VqIpLjI5VwghhHB3lSlxkaEiIYQQQrgNqbgIIYQQbk4qLkIIIYQQLkgqLkIIIYSbk6tDCyGEEEK4IKm4CCGEEG6uMs1xkcRFCCGEcHOVKXGRoSIhhBBCuA2puAghhBBuzlCJZudKxUUIIYQQbkMqLkIIIYSbq0xzXCRxEUIIIdxcZUpcZKhICCGEEG5DKi5CCCGEm5OKixBCCCGEC5KKixBCCOHmKtHR0JK4CCGEEO5OhoqEEEIIIVyQVFyEEEIIN2eoRGWIStRVIYQQQrg7qbgIIYQQbq4yzXEpN3FRSlUH3gaitdaDlVLNgK5a66/tHp0QQgghRDHWDBX9F/gLiLYsHwLG2ykeIYQQQlwlg8Fg85ursiZxidBazwWMAFrrfKDArlEJIYQQwmoGg+1vrsqaOS6ZSqlwwASglOoCpNo1Kiud27mP7d/Nx2Q00qBvd5oNGVhie0FeHhv/8x3Jx0/iHRhAtycfJjAynMQjMWz5ara5kQla3HkjtTq2cXwHKshkMvHHjAUc2rIfLx8vhj4ziuhGtUu1O3P4FAs++IH8nDyadGzGjWOGYjAYOHf0NIs+nUt+Xj4enh7cMnYYtVRdJ/Tk6plMJj74969sWHMAH19vXn1jBNc1q1Wq3fgxX5CQkEZBgZE27Rrw7EtD8fT04OXnvuNkTDwA6enZBAX5MWveM47uxjWZ8e5jDL6+LfGJaXQY8HyZbd6ffD+D+rYhKzuX0c98xs69MQCMurMXE564DYCpn/7KD/NXOyhq2zKZTEx/dyGb1pqf/+cn30WTpqWf/wljvyTR8vy3bFufcRPMz/+qpbuY+fkSTh6PY/qscahmpd83rspkMvHdR7+wa8MBvH29eezlkdRXpfs+9/M/WPPnVjLTs/jm76mF6+Njk/jy7TmkpWQQWNWff00cRXi1EAf2oGISdu9Dz56LyWikZq/u1L/5hhLbjXl57P3yv6TFnMQrMIBW/3oEv8gIjPn5HPjvD6TFnACDAXX3cMKaKif1QlSENRWXp4FFQEOl1DrgO+AJu0ZlBaPRyNZv59L7hbEMfu9VTq7fSurpcyXaHFuxAe8Af27+aDLqxn7smv0rAMG1oxn41gvcMPUlek8Yy5avfsRY4H5FpMNb9pN4Np7xX7/CkHEjWDxtXpntFk+by23jRjD+61dIPBvP4a0HAPjr60X0HXUDY6c/z/X3DOavrxc5MvwK2bD2IKdOJDDvtxd5ceIw3nnz5zLbvfXefXw//1lmL3iO5KQMli/ZZV7/7n3MmvcMs+Y9Q9/+rehzfUtHhl8hs+atYsh9Uy+7fVDfNjSsF0WLXk/x+IQv+eSthwEIDQ7g5fFD6XXrq/S89VVeHj+UkOAAR4VtU5vXHeT0yXi+WziBp1+5k4+nlP38v/rve/lyzjN8Pe9ZUpMzWfW3+fmv1zCKye/dT6t29R0Ztk3s2nCA2NMJvD/nJR5+fhjfvje/zHZtuzfj9S/Hl1o/e9pietzQganfPcftDw5kzozf7Ryx7ZiMRg7O+pG2Tz9Ot7cnEbtpCxlnzpZoc2b1Oqr4+9PjnTeoO/B6Ds/7xbx+5VoAur45kfbPPcmhn37GZDQ6vA/2UpkqLuUmLlrr7UBvoBvwGNBca73b3oGVJ+lIDEFRkQRWj8CzShXqdG3Pma0lwzqzbTf1e3UGoHbntpzfqzGZTFTx8cbD0xMwV2XAhZ+hKziwcS9tru+IwWCgdtN6ZGdkk55UshiWnpRKTtYFajeth8FgoM31HTmwYQ9gHhPNyboAwIWsCwSFV3V4H67V6hV7ufGW9hgMBlq0rktGejYJ8Wml2gUE+gJQkG8kL6+g1FNtMplY9tdOBgxu64iwbWLd5oMkpWRcdvvNA9sz++c1AGzecYTgqv5EVQthQO/WLFuzh+TUTFJSM1m2Zg8De7d2VNg2tW7lPgbe3AGDwUCzVnXJSL9AYrnPfz4GywugboPq1K5XzaEx28q2tXvpeYO5741b1CMrPZvkhNJ9b9yiHqERpd/TZ47H0rx9IwCatWvEtjV77R6zraQei8G/ejX8q0XiUaUKUZ07Er+j5Od+/I7dRPfoCkC1ju1I2n8Qk8lExtlzhFoqLN5Vq+Ll72euvgi3Y81RRUMvWdVEKZUK7NFax9knrPJlJ6fgHx5auOwXHkLSkZiSbZKK2nh4euLl70dueiY+VQNJPHKcTTO+JyshiS5j7y9MZNxJWmIKwREhhcvBEcGkJaQSFBZc1CYhlarF2lSNCCEtMQWAwY/dznevfMafXy3EZDIx+v3xjgncBuLjUqkWFVK4XK16MPFxqURElv6gfnLM5+zfc4quPa6j34CSX9Q7tx0jLDyIOnUj7R2yw0RHhXH6XGLh8pnYJKKjwoiOCuX02aSi9eeSiI4KLeshXF5CXCqR1UMKlyOrBZMQn0p4Gc//C//3BQf3naJT9+vo1b+VA6O0j6T4tBJDO2HVQkiOTy0zSSlLncbRbFm1hxuG92Lrqj1cyMohPTWTIDeovuUkJ+MTVvSa9QkNIe3Y8RJtLiSn4BtW9Llfxc+PvIxMgurUIn7HbqK6dCQnKZm0mJNcSEwmuIH7Vd3K4soVEluzZqjoYeArYJTl9iXwArBOKXWvHWOzq/BG9bnxvVcZ8NYL7F+4hILcPGeH5HBbfl/H4NG389ysyQwefTu/fPSjs0Oyi49nPMZvyyeRm5vP1s2HS2xb8r8dblVtEVfv3/8ZzbwlE8nLzWfHliPODsfpRo29lQM7jvLSA+9zYOdRQiOD8fD455+LNLpnN3zDQtj02hT07LkEN26A4R/Ubw+D7W+uyprJuVWAplrr81B4XpfvgM7AamCW/cK7PL/QELISkwuXsxNT8AsNKdkmzNzGPzwUY0EBeVnZeAeV/FURXDOKKj4+pJ46S1hD15+YumnxGrb+uQGAmk3qkJqQUrgtNSGVqhHBJdpXjQgmrVibtIQUqoaHALDj783cOMZcUGvRsw0LXTxxmf/TWhb+vAmAps1rExebUrgt7nwqkdWCL3NP8PHxolffFqxZsY/OXc3l4vz8AlYu28PMn56ya9yOdjY2iVo1wguXa0aFcTY2ibOxyfTs2rRofY0w1mw44IwQr8mvc9bxxy/m5181r038+ZTCbeZq2+Wff28fL7r1ac76lXvp0KWJvUO1uSU/r2XFoo0ANGham8S4lMJtSXEphF6h75cKjQzmqSkPAnAhK4fNK3cTEORn03jtxSc0lJykos/9nOQUfEJLVg19Q0O4kJSMb5j5cz8/OxuvwAAMlgm5F21+8x38o9xzuLCysybdrH0xabGIs6xLApxWpghrWJf02Dgy4hIoyM/n5IZt1GxfcoJlzfYtOb7a/EF3atMOqjdvgsFgICMuoXAybmZ8ImlnzxMQGV5qH66o8y09GTv9ecZOf56mXVuyc9kWTCYTpw7E4BvgW2KYCCAoLBgff19OHYjBZDKxc9kWmnZpYd4WHkzMHvMv0GM7DxFe07WHS+4c0aNwQm3vfi34Y/E2TCYTe3edIDDIt9QwUVZWTuG8l/z8Atat2U/d+kUfVFs2HqZe/Wolhpz+CX5fup277+gJQKe2jUhLzyI2LoWlq3bRv2crQoIDCAkOoH/PVixdtcvJ0Vrvtru688VPT/PFT0/TvU9zlvy2FZPJxP7dJwgI9C01TJSdlVM476Ugv4BNaw5Qx03ntQy8owdTZj7LlJnP0qFXS9b8ae774b0x+AX6Wj1MBJCekoHRMil10axl9Lmpk73Ctrmq9euSdT6O7PgEjPn5xG7aQmTbksN/kW1acXat+cdd3JbthDVVGAwGCnJyKcjJASBx734MHh4E1owutQ93JRWXklYqpX4DLh6ycodlXQCQYq/AyuPh6Un7B4azasp0jEYjDfp0Jbh2NHvm/UZY/TrU7NCKBn26sfE/M/lt/CTz4dBPPARAvD7KgYVL8KjiicHgQYeH7sKnaqCzunLNmnRsxqEt+/nwoTfw8vVm6FN3F26bPvYdxk43Hyp7y9hhLPjgB/Ish0M37tgMgNvG3cUfny+goMCIl7cXt44b4ZR+XItuPZuyfs0B7rxpCr6+XrzyRlHs9w57n1nzniE7O5fnxn1Dbm4+JqOJdp0acvuwroXtlv7pnsNEMz99gp5dmxIRGsSRTdN444P5eHmZ38pfff83fy7fwaC+bdi35iOysnN47NnPAUhOzWTKJ7+wdvGbALz98QKSUzOd1o+K6NyjKZvWHuTeIVPx9fXiudfuKtw2esQHfPHT02Rn5/LqU9+Qm1uAyWSkTYdG3HKn+flfu3wPn77zK6nJGbw07msaNYnm3/8Z7azuXJU2XZuyc8MBnh7+Nt6+Xjz20sjCbS/e/x5TZj4LwOzpi1m/dDu5F/J4/LbJ9L2lM3c8fAP7dxxlzozfMRgMXNe6AQ88c4ezunLVPDw9Uffcxfb3PsFkNBLdsxuBNaM5smARVevXpVrb1kT36s7eL75l7fOv4hXgT8t/PQJAbloa29//FIPBgE9oCC1GP+jk3vwzKKVuAD4GPIGvtNZTL9n+NPAIkA/EAw9prSs0K9pgMpnKC8oADAV6WFYlA9W11mOt3Idp0va/rz1CNza5XX8A5h7708mROMfwBubzKyTn/ObkSJwj1OdmAPzqjCyn5T9P9knzsOPpzMVOjsQ5agXcAsDWBPc51NiWOkTcBMDjG1Y4ORLnmNa1Lzj4cNVBf6298pf5NfhrUI8r9kEp5Yn5bPoDgNPAFmCk1np/sTZ9gU1a6yyl1L+APlrru8p8QCtZczi0CTiGOVu6HegLuM/AuBBCCPEP56Shok7AEa31Ma11LvATMKR4A631Cq11lmVxI1D6bIlX6bJDRUqpJsBIyy0BmAMYtNZ9K7pTIYQQQri9msCpYsunMR+4czkPA/+r6E6vNMflILAGuFlrfQRAKfXPOvxCCCGE+Aewx4HdSqnRQPHJX19orb+4xse6B+iA+YS2FXKlxGUoMAJYoZT6E3MJyIXnGQshhBDCVixJypUSlTNA8Qt91bKsK0Ep1R94Geittc6paFyXTVy01r8Cv1qOHhoCjAeqKaU+A37RWi+p6M6FEEIIUXEeBpvPzbXGFqCxUqo+5oRlBHB38QZKqbbA58ANtjrbvjWTczO11rO11rdgzqZ2YD5zrhBCCCEqKa11PvA48Bfmg3bmaq33KaVeV0rdamn2LhAIzFNK7VRKVfhqvtacx6V4kMmYy0bXNMYlhBBCCNtz1gnjtNZ/AH9csm5isb/723qfV5W4CCGEEML1/HOuulS+ytRXIYQQQrg5qbgIIYQQbs6Vry1ka1JxEUIIIYTbkIqLEEII4eYMzjkc2ikkcRFCCCHcnAwVCSGEEEK4IKm4CCGEEG6uMlUhKlNfhRBCCOHmpOIihBBCuDknXavIKSRxEUIIIdycTM4VQgghhHBBUnERQggh3FxlqkJUpr4KIYQQws1JxUUIIYRwczLHRQghhBDCBUnFRQghhHBzcji0EEIIIdxGZRoqMphMds/SKk8aKIQQQpg5NJV4ZO1Km3/XftWjj0umQ1JxEUIIIdxcZZqw6pDEZe6xPx2xG5czvMENAHy8b4mTI3GOJ5sPBOD5zcudHIlzvNOpHwCnMxc7ORLHqxVwCwB+dUY6ORLnyD75IwCNZqx2ciTOcWRMLwDuWbXKyZE4x/e9ezs7hH80qbgIIYQQbk4m5wohhBDCbVSmybmVaVhMCCGEEG5OKi5CCCGEm5OKixBCCCGEC5KKixBCCOHmKlMVQhIXIYQQws1VpqOKKlOSJoQQQgg3JxUXIYQQws3J5FwhhBBCCBckFRchhBDCzVWmKkRl6qsQQggh3JxUXIQQQgg3V5nmuEjiIoQQQrg5gxwOLYQQQgjheqTiIoQQQri5yjRUJBUXIYQQQrgNqbgIIYQQbq4yVSGsTlyUUlFAJ8AEbNFax9otKiGEEEJYTa5VdAml1CPAZmAocCewUSn1kD0DE0IIIYS4lLUVl+eAtlrrRAClVDiwHvjGXoEJIYQQwjoyObe0RCC92HK6ZZ0QQgghhMNYW3E5AmxSSi3EPMdlCLBbKfU0gNb6AzvFJ4QQQohyVKaKi7WJy1HL7aKFlv8H2TYcIYQQQlwtT2cH4EBWJS5a68kX/1ZKhQIpWuvKM4VZCCGEEC7hiomLUmoiMFdrfVAp5QP8D2gD5Cul7tZa/+2AGC/LZDLxx4wFHNqyHy8fL4Y+M4roRrVLtTtz+BQLPviB/Jw8mnRsxo1jhmIwGDh39DSLPp1Lfl4+Hp4e3DJ2GLVUXSf05OqZTCbWfv0zJ7bvo4qPN9c/fg+RDUv3Pe7oSZZ/+j35uXnUbdecHg/fgcFgYP3MX4nZugePKlUIrh5BvydG4RPg74SeXJvzu/exZ9ZcMJqo06c7TW4ZVGJ7QV4e2z+fSerxk3gFBtDx8Ufwjwzn1LrNHPljaWG7tFNn6PPGiwTXLf1v58pMJhPT313IprUH8PH15vnJd9Gkaa1S7SaM/ZLEhDQKCoy0bFufcROG4unpwaqlu5j5+RJOHo9j+qxxqGbu0/8Z7z7G4OvbEp+YRocBz5fZ5v3J9zOobxuysnMZ/cxn7NwbA8CoO3sx4YnbAJj66a/8MH+1g6K2rV61Q3mle0M8DQbmHojl852nSmwf2awG9zSPpsBkIiuvgFdWH+ZIchZVPAy83bsJzSMC8fQw8Ouh88zYceoye3FNqXv3cnLOHExGI5E9elBj8OAS29MPHeLknDlknTlDw0cfJax9+xLbC7Kz2TNpEqFt2lD37rsdGbpdyeHQRe4CtOXv+y3tI4HewNt2jMsqh7fsJ/FsPOO/foUh40aweNq8MtstnjaX28aNYPzXr5B4Np7DWw8A8NfXi+g76gbGTn+e6+8ZzF9fL3Jk+BVycvt+Us/FMWr6RPqMGcGqL+aU2W7153Po86+RjJo+kdRzcZzcsR+AWq0VIz56iREfvkhIdDW2/7y0zPu7IpPRyO6ZP9H1ucfp9++JnNmwhbQz50q0OblqPd4B/vR//3Ua3tCPfXN+AaB29070fetl+r71Mu3HPIB/ZLjbJS0Am9cd5PTJeL5bOIGnX7mTj6f8XGa7V/99L1/OeYav5z1LanImq/7eBUC9hlFMfu9+WrWr78iwbWLWvFUMuW/qZbcP6tuGhvWiaNHrKR6f8CWfvPUwAKHBAbw8fii9bn2Vnre+ysvjhxISHOCosG3GwwCv9WjEw7/v5YY5W7m5USSNQkv+6Fh8OI6b5m3j1vnb+XLnKV7q2gCAwQ0i8PY0cNO8bdz283ZGNKtBzSAfZ3TjmpiMRk7Mnk3jceNoMXkyiVu2kH32bIk23mFh1H/wQcI7dSrzMU4vXEhQkyaOCLdSUErdoJTSSqkjSqkJZWz3UUrNsWzfpJSqV9F9lpe45BYbEhoE/Ki1LtBaH8AFzrp7YONe2lzfEYPBQO2m9cjOyCY9KbVEm/SkVHKyLlC7aT0MBgNtru/IgQ17ADAYDORkXQDgQtYFgsKrOrwP1+r45j2oPp0wGAxEqfrkZmaTeUnfM5NSyc2+QJSqj8FgQPXpxPFN5r7XadMUD0/zqGj1JvXISExxdBeuWfLRGAKqRxJQLRKPKlWo2aUDsdt2lWhzbvsuavfoAkB0p3Yk7DuIyVTyF8npDVuo2aWDw+K2pXUr9zHw5g4YDAaatapLRvoFEuPTSrULCPQFoCDfSF5ePgbMM/jqNqhO7XrVHBqzrazbfJCklIzLbr95YHtm/7wGgM07jhBc1Z+oaiEM6N2aZWv2kJyaSUpqJsvW7GFg79aOCttmWlcL4kRaNqfSL5BnNPH70Xj61wsv0SYjr6Dwbz8vTy6+8k2AfxVPPA3g6+lBXoGRjNwC3EXm8eP4VKuGb6T5vR/WsSPJu0q+930iIvCvVQsMpWerZp44QX5aGsHNmjkqZIfxMNj+Vh6llCcwHRgMNANGKqUu/cd9GEjWWjcCPgT+XeG+lrM9RynVQikVCfQFlhTb5vRxhbTEFIIjQgqXgyOCSUso+eWdlpBK1WJtqkaEkGb5kh782O389fVC3r13En9+tZCBD9zigKhtIzMphcCI0MLlgPCQMhOXwPCQS9qklHqsA8s3Uqed+7yRLySn4BdW1He/sFAuJKeUbJOUgl+4uY2HpydV/P3Izcgs0ebMpm3UctPEJSEulcjqIYXLkdWCSYhPLbPtC//3BXf0fw3/AF969W/loAidJzoqjNPnis7WcCY2ieioMKKjQjl9Nqlo/bkkoqNCy3oIl1Y9wIdzGTmFy7EZOVQP8C7V7p7mNVg+siMvdGnA6+uOAPDnsQSy8gvYcF8XVt/Tma92nSY1J99hsVdUbkoK3mFhhcveISHkJSdbdV+T0cipefOoPWyYvcKrjDoBR7TWx7TWucBPmI86Lm4IMNPy93zgeqVUhY6BKi9xedKyo4PAh1rr4wBKqRuBHRXZsSvY8vs6Bo++nedmTWbw6Nv55aMfnR2Sw22d/xceHh406eWeX+DXKunIcTy9valau6azQ7G7f/9nNPOWTCQvN58dW444OxzhIN/vO0e/H7fwzsZjjG1nnrvXqloQBSboNmsTfX7YzMOta1E7yNfJkTpG3MqVBLdogXeo+yWr1nBGxQWoCRSfJHXasq7MNlrrfCAVCKcCrjjco7XeBFxXxvo/gD8qsuNrtWnxGrb+uQGAmk3qkJqQUrgtNSGVqhHBJdpXjQgmrVibtIQUqlqqEDv+3syNY4YC0KJnGxa6eOKy53+r2b90PQDVGtUhI6Hol0ZmYgoBYSX7HhAWXGIIyNwmpHD54PKNnNi6l1snP4GhjLKqq/INDSE7qajv2UnJ+IaGlGwTFkJ2YjJ+YaEYCwrIz8rGO7BoPsOZjVup1dW9krVf56zjj182AaCa1yb+fErhtvi4VCIigy9zT/D28aJbn+asX7mXDl3+2eP7Z2OTqFWj6HOxZlQYZ2OTOBubTM+uTYvW1whjzYYDzgixQs5n5lAjsGheSlSgD+czcy/b/rcj8bzeszEAtzaqxpqTSeQbTSRdyGNbbBotqwVyKv2C3eO2Be+QEHKTiqpmuSkpeFmZiGQcO0bG4cPErVqF8cIFjAUFePj6UnvoUHuF61CedvgIV0qNBkYXW/WF1voL2+/p6lg1T8Vyiv9JQA/Mw6RrgdcvXgLAkTrf0pPOt/QEQG/ex6bFa2jZux2nD57AN8CXoEu+vIPCgvHx9+XUgRhqXVeXncu20MVy/6DwYGL2HKF+q8Yc23mI8JqRju7OVWk5uBctB/cCIGbrXvb+bzWNerTn/KEYvP19y0xcvP18idXHqd6kHnrlZlre2BswT+7d8esybntjHF4+pcvMriykQV0yY+PIjEvALyyEMxu30v7/Sl46K6ptK06t3UhY4wac3bydiGaqMDkzGY2c3byNHq8844zwr9ltd3Xntru6A7BxzX5+nbOOvoPacGDPSQICfQmPLDlHKzsrh6zMHMIjq1KQX8CmNQdo2db9JuNerd+XbmfM/QOZu2g9ndo2Ii09i9i4FJau2sXk5+8qnJDbv2crJk79ycnRXr3dcenUDfajVpAv5zNzuKlhJE8vO1iiTd1gX06kmpORvnXDiEnNBuBsxgW61Azh18Nx+FXxoG21IP67+4zD+3CtAurVIycujpyEBLxCQkjasoWGjzxi1X2Lt0tYv57MmJh/TNJiL5Yk5UqJyhmg+NENtSzrympzWilVBQimgmfet3aC7U/AauAOy/IoYA7QvyI7r6gmHZtxaMt+PnzoDbx8vRn6VNGhbdPHvsPY6eZDJW8ZO4wFH/xAnuVw6MYdzfM5bht3F398voCCAiNe3l7cOm6EU/pxLeq2b87J7fv54f9ep4qPF/0ev6dw25ynp3LXB+bJ3b1G31V4OHSddk0L57Ks/moeBXn5LJo8HTBP0O0zxj367+HpSav7RrDh3U8xGY3U6dWNqrWiOfDzYkLq16FGu9bU7d2d7TP+y9/PTMQr0J8OYx8uvH+iPoJfWCgB1Vw7Ub2Szj2asmntQe4dMhVfXy+ee+2uwm2jR3zAFz89TXZ2Lq8+9Q25uQWYTEbadGjELXd2BWDt8j18+s6vpCZn8NK4r2nUJJp//2f05XbnUmZ++gQ9uzYlIjSII5um8cYH8/HyMn+UffX93/y5fAeD+rZh35qPyMrO4bFnPwcgOTWTKZ/8wtrFbwLw9scLSE7NvOx+XFWBCSavPcK3N7XA02Bgno7lcHIWT3aoy974dJadSOLeFjXpXjOEPKOJtJx8nl9hPjj0+71n+Xdfxf+Gt8cAzNfn0Unu829g8PSkzsiR6I8+AqORiO7d8YuO5szChfjXrUtomzZkxMRw5D//oSAri5TduzmzaBEtJ08u97HdnZPOnLsFaKyUqo85QRkBXHqM+SLMRyVvwHyR5uUVPQ+c4dIjLcqilNqrtW5xybo9WuuWVuzDNPfYn9can1sb3uAGAD7et6Sclv9MTzYfCMDzm5c7ORLneKdTPwBOZy52ciSOVyvAPNHdr85IJ0fiHNknzcPOjWa453liKurIGHNl+J5Vq5wciXN837s3gENTiQ/3LrX5iVyeajGg3D5Y5rx+hPnkvd9ord9SSr0ObNVaL1JK+QKzgLZAEjBCa32sInFZW3FZopQaAcy1LN8J/FWRHQshhBDCNpx1Arqy5rxqrScW+/sCYNNDuco7c2465jktBmA85qwJzJlVBvCsLYMRQgghxNWTiyxaaK3lIopCCCGEcBnlVVyus1ynqF1Z27XW2+0TlhBCCCGsJVeHLvI05mO43y+2rvhAWj+bRySEEEIIcRnlJS5fKaWitNZ9AZRS92M+JDoGeM2+oQkhhBDCGpVpjkt5p/yfAeQCKKV6AVMwX3MglSuflEYIIYQQDuJhMNn85qrKq7h4aq0vnl/5Lsyn+/0Z+FkptdOukQkhhBBCXKK8ioun5RS9ANcDxc8kZu05YIQQQghhR54G299cVXnJx4/AKqVUApANrAFQSjXCPFwkhBBCCOEw5Z3H5S2l1DKgBrCk2PUFPIAn7B2cEEIIIcpXmSbnljvco7XeWMa6Q/YJRwghhBDi8mSeihBCCOHmpOIihBBCCLdRmRKX8o4qEkIIIYRwGVJxEUIIIdycpwufMM7WpOIihBBCCLchFRchhBDCzVWmKoQkLkIIIYSbk8m5QgghhBAuSCouQgghhJuTiosQQgghhAuSiosQQgjh5irT4dCSuAghhBBuToaKhBBCCCFckFRchBBCCDdXmSouBpPJ7uNilWfgTQghhDBzaCqx+OT/bP5de0udwS6ZDknFRQghhHBzlani4pDE5cO9Sx2xG5fzVIsBADy+YYWTI3GOaV37AjB5x99OjsQ5JrXtD8DWhN+dHInjdYi4CYBGM1Y7ORLnODKmFwB+dUY6ORLnyD75IwCD/lrr5Eic469BPRy+T89KlLjI5FwhhBBCuA0ZKhJCCCHcnEclOo+LVFyEEEII4Tak4iKEEEK4ucpUhahMfRVCCCGEm5OKixBCCOHm5HBoIYQQQrgNORxaCCGEEMIFScVFCCGEcHNyOLQQQgghhAuSiosQQgjh5mRyrhBCCCHcRmVKXGSoSAghhBBuQyouQgghhJurTFWIytRXIYQQQrg5qbgIIYQQbs5Qiea4SOIihBBCuLlKlLfIUJEQQggh3IdVFRellA9wB1Cv+H201q/bJywhhBBCWEuGikpbCKQC24Ac+4UjhBBCCHenlAoD5mAueMQAw7XWyZe0aQN8BlQFCoC3tNZzyntsaxOXWlrrG6wPWQghhBCO4oLzPiYAy7TWU5VSEyzLL1zSJgu4T2t9WCkVDWxTSv2ltU650gNb29f1SqmWVxu1EEIIISqlIcBMy98zgdsubaC1PqS1Pmz5+ywQB0SW98BXrLgopfYAJku7B5VSxzAPFRkAk9a6lfV9EEIIIYQ9GFzv6tDVtdbnLH/HAtWv1Fgp1QnwBo6W98DlDRXdbFV4QgghhHAae8zNVUqNBkYXW/WF1vqLYtv/BqLKuOvLxRe01ial1GUzK6VUDWAWcL/W2lheXFdMXLTWJywP2gXYp7VOtyxXBZoCJ8rbgT2ZTCbWfTOfk9v3UcXbm75P3Etkg9ql2sUfPcmKabPIz82jTrvmdH/oTgwGA0fXb2frnD9IPnOeoVOfpVqjuk7oxbVJ2L0PPXsuJqORmr26U//mklOQkvVh9Oy5ZJw6Q8t/PUz1ju0Lt51du4Fji/8AoMEtNxLdo6tDY7eFszv3sW3mfExGIw37daf5kIElthfk5bFh+nckHT+JT2AA3Z98mMBq4ZzbfYCdPy7EmF+ARxVP2o66nagWykm9uHYmk4nvPvqFXRsO4O3rzWMvj6S+qlWq3dzP/2DNn1vJTM/im7+nFq6Pj03iy7fnkJaSQWBVf/41cRTh1UIc2IOK6VU7lFe6N8TTYGDugVg+33mqxPaRzWpwT/NoCkwmsvIKeGX1YY4kZ1HFw8DbvZvQPCIQTw8Dvx46z4wdpy6zF9c0493HGHx9W+IT0+gw4Pky27w/+X4G9W1DVnYuo5/5jJ17YwAYdWcvJjxxGwBTP/2VH+avdlDUttMhIoQx1zXA02Dgf6fPM/f46RLbh9aN5oZaURSYTKTm5vHB3sPEXTAfU/JW++ZcFxzEvuQ0Ju7Y74zw3YolSfniCtv7X26bUuq8UqqG1vqcJTGJu0y7qsDvwMta643WxGXtHJfPgIxiyxmWdU51cvt+Us/FM3LaJHr/ayRrvvipzHarv5hD73/dzchpk0g9F88pyws2rE40g55/lBrNGjoy7AozGY0cnPUjbZ9+nG5vTyJ20xYyzpwt0cY3LJTmj9xPVJeOJdbnZWRybOHvdH51Ap0nTuDYwt/Jy8x0ZPgVZjQa2frNXPpOGMtN77/KiXVbST19rkSboys24B3oz60fT0bd1I+ds38FwCcokN7PjeGmd1+m6//dx4bpM8vYg+vbteEAsacTeH/OSzz8/DC+fW9+me3adm/G61+OL7V+9rTF9LihA1O/e47bHxzInBm/2zli2/EwwGs9GvHw73u5Yc5Wbm4USaNQ/xJtFh+O46Z527h1/na+3HmKl7o2AGBwgwi8PQ3cNG8bt/28nRHNalAzyMcZ3bhms+atYsh9Uy+7fVDfNjSsF0WLXk/x+IQv+eSthwEIDQ7g5fFD6XXrq/S89VVeHj+UkOAAR4VtEx7A2KYNeWXbPh5du52+NSKpE+BXos3R9Eye2LCTf63fwdrzCTzSpF7htnnHT/POnkOODdpBDAbb3ypoEXC/5e/7MR+dXIJSyhv4BfhOa132h1gZrE1cDFrrwjKPpZTj9LPuxmzZTZPenTAYDFRvUp+czGwyk1NLtMlMTiUv6wLVm9THYDDQpHcnjm/eDUBorShCal5x2M0lpR6Lwb96NfyrReJRpQpRnTsSv2N3iTZ+kREE1a5V6tWXsHc/Yc2b4hUYgFdAAGHNm5Kwx71+eSQeiSEwKpLA6hF4VqlC3W7tOb21ZP9Pb91N/V6dAajTuS3n92lMJhNh9WvjHxYCQHCtGhTk5lGQl+foLlTYtrV76XlDBwwGA41b1CMrPZvkhLRS7Rq3qEdoRNVS688cj6V5+0YANGvXiG1r9to9ZltpXS2IE2nZnEq/QJ7RxO9H4+lfL7xEm4y8gsK//bw8ufjhZQL8q3jiaQBfTw/yCoxk5BbgTtZtPkhSSsZlt988sD2zf14DwOYdRwiu6k9UtRAG9G7NsjV7SE7NJCU1k2Vr9jCwd2tHhW0TKjiIs1kXiM3OId9kYuW5eLpWK/nc70pKJcdoHm04kJJOhG9RYrozKZXsfPd6vt3YVGCAUuow0N+yjFKqg1LqK0ub4UAv4AGl1E7LrU15D2xt8nFMKTWOoirL/wHHrqIDdpGZlEJgRGjhcmB4CJmJKQSEBhe1SUwhIDykZJukFAdGaXs5ycn4hBX12yc0hLRjx62+r2+x+/qGhpCTnHyFe7ie7KQUAsKL+uAfFkLCkZjLtvHw9MTLz4+c9Ex8qwYWtjm1aQeh9Wvj6eXlkLhtKSk+rcTQTli1EJLjU8tMUspSp3E0W1bt4Ybhvdi6ag8XsnJIT80kyA1+gVcP8OFcRtHppGIzcmhdPahUu3ua1+ChVrXw8vTgnsW7APjzWAL964Wz4b4u+Fbx5K31R0nNyXdY7I4QHRXG6XOJhctnYpOIjgojOiqU02eTitafSyI6KrSsh3BZ4b7exF8oeu4TLuRwXUjp5/6iG2pVZ0uCe32+XStXO/+c1joRuL6M9VuBRyx/fw98f7WPbW3FZQzQDTgDnAY6U3LCjhBuJeXUWXbOXkinR0Y6OxSnGDX2Vg7sOMpLD7zPgZ1HCY0MxsPDBc8EUQHf7ztHvx+38M7GY4xtZ56/1qpaEAUm6DZrE31+2MzDrWtRO8jXyZEKe+hXI5LGVQOZf8kcmH8qD4Ptb66q3IqLUsoT+FBrPcIB8ZRr7/9WceDv9QBENqpLRrFsOuOS6gpAgKUKU6JNWMk27sYnNJScpKJ+5ySn4BNq3S8nn9BQkg8WjfFeSE4h9LomNo/RnvzCQshMLOp/VlJK4fDPpW38w0MxFhSQl52NT5C5mpCVmMya97+k69j7CIoq95QBLmPJz2tZscg8d61B09okxqUUbkuKSyE0Mvgy9ywtNDKYp6Y8CMCFrBw2r9xNQJBfOfdyDeczc6gRWFT+jwr04Xxm7mXb/3Ykntd7Ngbg1kbVWHMyiXyjiaQLeWyLTaNltUBOpV+we9yOcjY2iVo1ioZPakaFcTY2ibOxyfTs2rRofY0w1mw44IwQr1nihVwiiw39RPj6kHCh9HPfNiyYkQ1q8+yWPeSZXO4wYVFB5f7E0loXAHUtk2icrsXg3gx7/0WGvf8i9Tu14tCqzZhMJs4fOo63v1+JYSKAgNBgvPx9OX/oOCaTiUOrNlOvo3uffqZq/bpknY8jOz4BY34+sZu2ENnWuj5FtGhG4t795GVmkpeZSeLe/US0aGbniG0rvGFd0mPjyIhLoCA/nxPrt1GzfcnzI9Zq35LjqzcBcHLTDqo3b4LBYCA3M4uV//6MNncPIVK516TsgXf0YMrMZ5ky81k69GrJmj+3YjKZOLw3Br9AX6uHiQDSUzIwWuYBLJq1jD43dbJX2Da3Oy6dusF+1AryxcvDwE0NI1kWk1iiTd3goipK37phxKRmA3A24wJdaoYA4FfFg7bVgjianO2w2B3h96XbufuOngB0atuItPQsYuNSWLpqF/17tiIkOICQ4AD692zF0lW7nBzt1dFp6dT096O6nw9VDAb61IhkY1xSiTYNgwIY17wRk3bsJzXX/eavXSuDHW6uyuo5LsA6pdQioPAQFK31B3aJykp12jXn5PZ9/Dh2MlV8vOgz9p7CbfOemcKw918EoOejw1kx7XsKcvOo3bYZddqZv6iPb9rF2q/mkZ2Wwf/enkF4vZrcPPFxp/Tlanh4eqLuuYvt732CyWgkumc3AmtGc2TBIqrWr0u1tq1JPRbDrk9nkJeZRcLOPRz95Te6vT0Jr8AAGtx6I5smm49KaDDkJrwCXX9eQ3Eenp50eHA4K96ejslopEHfroTUjmb33N8Ia1CHWh1a0bBvN9ZPn8miJyfhHRhAj3EPAXDor1Wkn49nz89/sOdn8yHh/V56At/gy4+Tu6I2XZuyc8MBnh7+Nt6+Xjz2UtGQ14v3v8eUmc8CMHv6YtYv3U7uhTwev20yfW/pzB0P38D+HUeZM+N3DAYD17VuwAPP3OGsrly1AhNMXnuEb29qgafBwDwdy+HkLJ7sUJe98eksO5HEvS1q0r1mCHlGE2k5+Ty/QgPw/d6z/Luv4n/D22MA5uvz6CT3Oqpu5qdP0LNrUyJCgziyaRpvfDAfLy/zR/lX3//Nn8t3MKhvG/at+Yis7Bwee/ZzAJJTM5nyyS+sXfwmAG9/vIDkVPfqu9EE0w8c5e32LfAwwJIz5zmRmcV9jepwKDWDjfFJPKrq4+fpySutrwMg7kIOr+0wV5be79SSWgH++Hl68H3vjny49zDbilXkhXswmKwooymlJpW1Xms92Yp9mD7cu/Rq4/pHeKrFAAAe37DCyZE4x7SufQGYvONvJ0fiHJPamk9xsDXBfQ41tpUOETcB0GiG+50nxBaOjOkFgF+dyjmHKvvkjwAM+mutkyNxjr8G9QAHFy32p/xm8zGxZiE3u2ThxaqKi5UJihBCCCGcwCUzDDuxKnFRSkUCzwPNgcLBY611PzvFJYQQQghRirXHP/4AHATqA5OBGGCLnWISQgghxFWoTJNzrU1cwrXWXwN5WutVWuuHAKm2CCGEEMKhrD2q6OIxZeeUUjcBZ4Ew+4QkhBBCiKvhyieMszVrE5c3lVLBwDPAp0BV4Cm7RSWEEEIIUYYrJi5KKV/Mp/tvBNQEvtZa93VEYEIIIYSwTiUquJRbcZmJeZhoDTAYaAY8ae+ghBBCCGE9g6HyXNqgvMSlmda6JYBS6mtgs/1DEkIIIYQoW3lHFRVe6EFr/c+69rsQQgjxD1GZDocur+LSWimVZvnbAPhZlg2ASWtt/VXdhBBCCCEq6IqJi9ba01GBCCGEEOLaGFy5RGJj1h4OLYQQQggXZe3ZZP8JKlNfhRBCCOHmpOIihBBCuLnKNFQkFRchhBBCuA2puAghhBBurhIVXCRxEUIIIdydDBUJIYQQQrggqbgIIYQQbq4SFVyk4iKEEEII9yEVFyGEEMLNeVSikotUXIQQQgjhNqTiIoQQQri5SlRwkcRFCCGEcHcGg8nZITiMDBUJIYQQwm0YTCa7Z2mVJw0UQgghzBw6enM+e5HNv2ur+93qkiNQUnERQgghhNtwyByXqbuWOmI3LmdC6wEAdFuw1smROMf6oT0sfx1yahzO0wSAxzescHIcjjeta18A7lm1ysmROMf3vXsDMOivyvne/2uQ+b3vV2ekkyNxjuyTPzp8n5XplP8yOVcIIYRwc5Uob5GhIiGEEEK4D6m4CCGEEG6uMlUhKlNfhRBCCOHmpOIihBBCuDmZnCuEEEIIN1J5MhcZKhJCCCGE25CKixBCCOHmDFJxEUIIIYRwPVJxEUIIIdycwVB56hCVp6dCCCGEcHtScRFCCCHcXuWZ4yKJixBCCOHmKtPkXElchBBCCGFTSqkwYA5QD4gBhmutky/TtiqwH/hVa/14eY8tc1yEEEIIt2eww61CJgDLtNaNgWWW5ct5A1ht7QNL4iKEEEIIWxsCzLT8PRO4raxGSqn2QHVgibUPLENFQgghhJtzwcOhq2utz1n+jsWcnJSglPIA3gfuAfpb+8CSuAghhBBuz/aTc5VSo4HRxVZ9obX+otj2v4GoMu76cvEFrbVJKWUqo93/AX9orU8rpayOSxIXIYQQQpRiSVK+uML2y1ZJlFLnlVI1tNbnlFI1gLgymnUFeiql/g8IBLyVUhla6yvNh5HERQghhHB3Lng49CLgfmCq5f8LL22gtR518W+l1ANAh/KSFpDJuUIIIYSwvanAAKXUYczzV6YCKKU6KKW+qsgDW11xUUr1ABprrb9VSkUCgVrr4xXZuRBCCCEqztUqLlrrROD6MtZvBR4pY/1/gf9a89hWJS5KqUlAB0AB3wJewPdAd2vuL4QQQgh7qjwDKNb29HbgViATQGt9FgiyV1BCCCGEEGWxdqgot/jhTEqpADvGZLXTO/ez6dv5mIxGmlzfjVa3DSyxvSAvj9XTZpF47CQ+QQH0Gf8QQdXCuZCewYoPvibhyAka9elC14eHO6kH165z9RDGt2qAp8HA4pjzzDp0usT2EY2iuaVeFAUmEyk5eby97TCx2Tm0iwhmXKv6he3qBvkzafNBVp9LcnQXKsRkMvHWW1+watU2fH19mDr1SZo3b1Sq3b33vkhcXDK+vt4AfPPN64SHh7Bgwd+88863VK8eDsA999zEsGGDHNqHikjYvQ89ey4mo5GavbpT/+YbSmw35uWx98v/khZzEq/AAFr96xH8IiMw5udz4L8/kBZzAgwG1N3DCWtq/WGIriJ1715OzpmDyWgkskcPagweXGJ7+qFDnJwzh6wzZ2j46KOEtW9fYntBdjZ7Jk0itE0b6t59tyNDr7AOESGMuc783v/f6fPMPV7yvT+0bjQ31DK/91Nz8/hg72HiLuQA8Fb75lwXHMS+5DQm7tjvjPArbMa7jzH4+rbEJ6bRYcDzZbZ5f/L9DOrbhqzsXEY/8xk798YAMOrOXkx44jYApn76Kz/Mt/pkrS7PYHCtoSJ7sjZxmauU+hwIUUo9CjwEfGm/sMpnNBrZ+PVcBr3yOP7hISx+8V3qdGhJSK0ahW0OLd+AT4Afd376GsfWbWXrDwvp+9RDeHp50e6um0k+eZbkU+eusBfX5AE827ohT67dS1x2Ll/3bcOac4nEpGcXtjmUkslDK3aSU2Dk9vpR/F/LekzcrNmekMoDy3cCEORVhXmD2rMpLsUp/aiI1au3ERNzliVLPmfXLs1rr33GvHnvl9n2vfeeoWXLxqXW33hjTyZOHGPvUG3OZDRycNaPtHvuSXzDQtk0eQqRbVsRWDO6sM2Z1euo4u9Pj3feIHbjFg7P+4VW//coZ1auBaDrmxPJTUtj+/vT6DxpAgYP9ykzm4xGTsyeTZOnnsI7NJT9b79NSOvW+EUX9d87LIz6Dz5I7JKyT8Z5euFCgpo0cVTINuMBjG3akBe37iXhQi6fdm3DxrhETmYWvfePpmfyxIad5BiN3Fw7ikea1OPt3RqAecdP4+PpyU21yjr1hnuYNW8VM2b+xVcf/l+Z2wf1bUPDelG06PUUndo24pO3HqbXkFcJDQ7g5fFD6X7Ty5iA9b+/xe9Lt5GSmunYDogKK/fTSillwHyhpPnAz5jnuUzUWn9q59iuKOFIDEFREQRVj8CzShUadGvHyS27S7Q5uXU3jfp0BqBel7ac26sxmUx4+fpQ/bqGeHp7OSP0CmsWFsTpzAuczcoh32Ti79Px9KwRXqLN9oRUcgqMAOxLSqean0+px+lXM5wNscmF7dzJsmUbue22fhgMBtq0uY60tEzi4tyranStUo/F4F+9Gv7VIvGoUoWozh2J31HytR+/YzfRPboCUK1jO5L2H8RkMpFx9hyhlgqLd9WqePn7masvbiTz+HF8qlXDN9Lc/7COHUnetatEG5+ICPxr1YIyfoVmnjhBfloawc2aOSpkm1HBQZzNukBstvm9v/JcPF2rlXzv70pKJcdofk8fSEknwrfovb8zKZXs/AKHxmxr6zYfJCkl47Lbbx7Yntk/rwFg844jBFf1J6paCAN6t2bZmj0kp2aSkprJsjV7GNi7taPCdgCXu1aR3ZSbuGitTZjPbLdUa/2c1vpZrfVSB8R2RVlJqQSEhxYu+4eHkpmUetk2Hp6eePv7kZPu/tl1pK8357NzCpfjs3OI9PO+bPub61VnY2zpi3L2rxXJ0tPxdonR3s6fTyQqKqJwOSoqnPPnE8ts+9JLHzNkyDimT/8Jk6no5I1LlqznllueYNy4KZw75z7/DjnJyfiEFb32fUJDyEku+fxeSE7BN6zotV/Fz4+8jEyC6tQifsdujAUFZMcnkBZzkguJZV6w1WXlpqTgHRZWuOwdEkJesnV9MBmNnJo3j9rDhtkrPLsK9/Um/kLRez/hQg4Rvpd/799QqzpbEtzr+a2o6KgwTp8r+iw4E5tEdFQY0VGhnD5b9OPmzLkkoqNCy3oI4eKsrQ9vV0p1tGskwi4G1Y7kutBAfjhcchw83NeLBsEBbDqf4pzAHOS9955l8eJp/PDDVLZt28fChSsA6Nu3E8uXf83ixZ/SrVsbXnjhI+cG6iDRPbvhGxbCptemoGfPJbhxA7caJqqouJUrCW7RAu/Qf/4XVr8akTSuGsj8S+bAiH8mgx3+c1XWznHpDIxSSp3AfGSRATBprVvZLbJy+IcFk1nsl2JWYjIBYcFltgkID8VYUEBuVjY+QS4xr7hC4i/kUr3Y0E+knw/x2bml2nWIDOZ+VZuxa/aQZyx5mYjra0ay+mwiBaayLh/hmn744Xfmzv0LgJYtGxMbm1C4LTY2sXCibXEX1wUG+nPzzb3ZvfsQt93Wj9DQqoVthg0byLvv/te+wduQT2goOUlFr/2c5BR8Lvki9g0N4UJSMr5h5td+fnY2XoEBGCwTci/a/OY7+EdVc1jstuAdEkJuUtEv59yUFLysTEQyjh0j4/Bh4latwnjhAsaCAjx8fak9dKi9wrWpxAu5RBYb+onw9SHhQun3ftuwYEY2qM2zW/aQ50bvcVs4G5tErWJD5zWjwjgbm8TZ2GR6dm1atL5GGGs2HHBGiHZSeX6AWNvTQUBDoB9wC3Cz5f9OE9GwLmnn4kmPS6AgP59j67dTu0PJPKpO+5YcWbkJgJiNO6jRvMk/Yub1geR0agX6UcPfhyoGA/1rRbL2kqOCmgQH8ELbRjy/YT/JOXmlHqN/7QiWnnKf4RGAUaNuYuHCT1i48BP69+/Cr78ux2QysXPnQYKC/KlWLaxE+/z8ApIsw4d5efmsXLmFxo3rApSYD7N8+WYaNqztuI5UUNX6dck6H0d2fALG/HxiN20hsm3J135km1acXbsBgLgt2wlrqjAYDBTk5FKQYx5qSNy7H4OHR4lJve4goF49cuLiyEkw9z9pyxZCW1s3V6HhI4/Q+t//pvWUKdQeNoyILl3cJmkB0Gnp1PT3o7qf+b3fp0YkGy+Z29UwKIBxzRsxacd+UnNLv/f/6X5fup277+gJQKe2jUhLzyI2LoWlq3bRv2crQoIDCAkOoH/PVixdtaucRxOuyKqKi9b6BIBSqhrga9eIrOTh6UmXh4az5K3pmIwmGvftQmjtGmyf8xsRDetQp0MrGvfrxppp3zH/idfwCQygz/gHC+8/b+xEcrMuYMzP5+SW3Qx6ZWyJI5JcWYEJPth5lA+7t8DTAL+dOM/x9CweaVqHgykZrD2XxNiW9fGr4smbna8D4Hx2Di9Yfl1E+ftQ3c+HHQmpV9qNS+vduwOrVm1lwIDR+Pn58PbbTxZuGzJkHAsXfkJubh6PPDKJvLwCjMYCunZtw/Dh5kPmZ81azPLlm/D09CQ4OIgpU5683K5cjoenJ+qeu9j+3ieYjEaie3YjsGY0RxYsomr9ulRr25roXt3Z+8W3rH3+VbwC/Gn5L/OJKs1HEn2KwWDAJzSEFqMfLGdvrsfg6UmdkSPRH30ERiMR3bvjFx3NmYUL8a9bl9A2bciIieHIf/5DQVYWKbt3c2bRIlpOnuzs0CvMaILpB47ydvsWeBhgyZnznMjM4r5GdTiUmsHG+CQeVfXx8/Tkldbm937chRxe22F+77/fqSW1Avzx8/Tg+94d+XDvYbYlpjixR1dv5qdP0LNrUyJCgziyaRpvfDAfLy/zV9lX3//Nn8t3MKhvG/at+Yis7Bwee/ZzAJJTM5nyyS+sXfwmAG9/vIDkf9ARRa48tGNrBpMVZUSl1K3A+0A05is81gUOaK2bW7EP09RdTp/L6xQTWg8AoNuCtU6OxDnWD+1h+euQU+NwHvPhto9vWOHkOBxvWte+ANyzapWTI3GO73v3BmDQX5Xzvf/XIPN736/OSCdH4hzZJ38EBx+Wk52/3uZjgn5VurlkNmTtUNEbQBfgkNa6PubrD2y0W1RCCCGEsJrBYLD5zVVZm7jkWS6Y5KGU8tBar8B87SIhhBBCOF3lOY+LtUcVpSilAoHVwA9KqTgs1y0SQgghhHCUK1ZclFJ1LH8OAbKAp4A/gaM4+agiIYQQQpgZ8LD5zVWVV3H5FWintc5USv2stb4DmGn/sIQQQgghSisvcSk+yNXAnoEIIYQQ4lq57pwUWysvcTFd5m8hhBBCuAhXPgrI1spLXForpdIwp3J+lr+h6JT/VS9/VyGEEEII27pi4qK19nRUIEIIIYS4VpWn4uK604aFEEIIIS5h7XlchBBCCOGiXPnwZVurPD0VQgghhNuTiosQQgjh9irPHBdJXIQQQgg3Z6hEiYsMFQkhhBDCbUjFRQghhHBzlekEdFJxEUIIIYTbkIqLEEII4fYqTx1CEhchhBDCzcnkXCGEEEIIFyQVFyGEEMLtScVFCCGEEMLlSMVFCCGEcHOV6XBoSVyEEEIIt1d5BlAqT0+FEEII4fak4iKEEEK4ucp0OLTBZDLZex9234EQQgjhYhycSRyyw3dtE5fMhhyRuAghhBBC2ITMcRFCCCGE25DERQghhBBuQxIXIYQQQrgNSVyEEEII4TYkcRFCCCGE25DERQghhBBuw6EnoFNKFQB7LPs9ANyvtc5yZAy2opRaCTyrtd56yfoHgA5a68ev8XFfBu4GCgAj8JjWelMFY70VaKa1nlqRx7E8VobWOrCij3OV+7T6daOUeg3I0Fq/57gInUcpdRvwC9BUa33QyeHYVVnvDeBR4AOt9f7LvTaVUl2AjwEfy22O1vo1hwVuA7b+7FRK1QN+01q3sE2EjlPs3+Ki27TWMU4KRziBoysu2VrrNpY3Sy4wxsH7twmllKedHrcrcDPQTmvdCugPnLLyvpdNQrXWi2yRtDjRP+J1YycjgbWW//9jXe69obV+RGu9v5y7zwRGa63bAC2AuXYN1j6u6T1wpc8FN3bx3+LiLaa8OyilDEopGWH4h3Dmi3oN0EopdQvwCuANJAKjtNbnlVK9Mf9KAvPZd3sBgcAcoCrm2P+ltV6jlBoITMb8a+oo8KDWOkMpFYP5Q+sWwAsYprU+qJSKBGYD0cAGYADQXmudoJS6BxhniWcT8H9a6wKlVAbwOeYPzLHFO6KUehB4EUgBdgE51/hvUgNI0FrnAGitEyyPH4O5ipOglOoAvKe17mOpLjQEGgAnlVL1gYe11vss91sJPIv5w7oD8DKwG6ivtTYqpQKAg5b71wGmA5FAFvCo5d+qvuXfKhBYeI39sqU1QCsApdR9mPtnAnZrre8t3lAp9SgwGvNzeQS4V2udpZQaBkzC/Ms9VWvdSynVHPjW0tYDuENrfdhBfbomSqlAoAfQF1gMTLJ8OE8D+mFOevOAb7TW85VS7YEPMD+XCcADWutzTgn+6l3uvbGSYpVPpdSHwEAgFhihtY4HqgHnLPcrAPZb2r6G+f3TCIgA3tFaf+m4Ll2z8j47X6Pk58J4YIZlGeBfwFnAUyn1JdANOAMM0VpnO7IjtmB5HywEQjF/zr+itV5oqSr9hflzvD1wo1JqODAc83fFL1rrSc6JWlSEUzJQy6+AwZjLfWuBLlrrtsBPwPOWZs8CYy2/knoC2ZjLxH9Z1rUGdiqlIjC/eftrrdsBW4Gni+0uwbL+M8tjgvlLa7nWujkwH/OXNkqppsBdQHfLPgqAUZb7BACbtNattdZri/WlBuakqTvmL5FmFfinWQLUVkodUkr9x5K8lacZ5r6PxJzUDS8WV43iQ1la61RgJ3DxcW/G/O+ZB3wBPKG1bo/53+k/ljYfA59prVti+fB3luKvG0ui8QrQT2vdGniyjLss0Fp3tGw/ADxsWT8RGGRZf6tl3RjgY8vz3gE4bb+e2MwQ4E+t9SEg0ZKYDAXqYX5d3At0BVBKeQGfAndanuNvgLecEfQ1sua9EQBstbyvV2F+nwN8CGil1C9KqceUUr7F7tMKc5LXFZiolIq2Yx8qzMrPTij5ufAJsMryem8H7LO0aQxMt/x7pQB3OKQTFeenlNppuf0CXABut3zO9wXeV0pdPFV9Y+A/lj4qy3InoA3QXinVy/Hhi4pydOLip5TaiTm5OAl8DdQC/lJK7QGeA5pb2q4DPlBKjQNCtNb5wBbgQcsvipZa63SgC+Y36TrLY98P1C22zwWW/2/D/IEO5gTjJwCt9Z9AsmX99Zgz8y2Wx7qeol8pBcDPZfSpM7BSax2vtc7FnDxcE611hmX/o4F4YI5lzsyVLCr2K2kucKfl7+GYk7JLzcGcnAGMsOwjEPOvrnmWfn+O+RcumBOyHy1/z7qa/thQWa+bfsC8i7+8tdZJZdyvhVJqjeW1NYqSr63/WioyF4f9NgAvKaVeAOq6yS/PkVhex5b/j8T82p6ntTZqrWOBFZbtCnPlbanl3/IVzO89t2Dle8NI0fvve8z/FmitX8ecjC7B/OPnz2L3Wai1zra8jlZg/lJzRVfz2QklPxf6Yf7hhta6wPIDBuC41nqn5e/in4+urvhQ0e2Yrwn0tlJqN/A3UBOobml7Qmu90fL3QMttB7AduA5zIiPcjKOHirItv2gLKaU+xTy5bpFSqg/wGoDWeqpS6nfgRsxJySCt9WpLhnwT5i+eDzAnHUstvyzKcnHYpoDy+2sAZmqtXyxj2wVLmdmuLPtYCay0fCDdD+RTlGT6XnKXzGL3PaOUSlRKtcKcnJQ1Dr4I85s8DPMXwXLMv1RTLn1uinH2Ba3Ket1Yc7//Yp64t8vyJdcHQGs9RinVGfPraJtSqr3WerZSapNl3R9Kqce01stt1wXbsjx//YCWSikT5gTMhHmiblkMwD6tdVcHhWhzl3lvXEnh61ZrfRT4zDI0Eq+UCr+0zWWWXYXVn50WmZSv+JB2AeBXwRidZRTmIe72Wus8y9D6xc/J4v8OBmCK1vpzB8cnbMwVJisFYx5fhWIfREqphlrrPVrrf2OutFynlKoLnLeMQ3+Fuey5EeiulGpkuV+AUqpJOftcR9GQykDMY6MAy4A7lVLVLNvCLPu8kk1Ab6VUuKUcP8yqXpdBmRX/BdAGOAHEYE4yoPxy7hzMJeNgrfXuSzdafrluwTwE9JvlF1gacNwy9+PiRLbWlrusw1yZgaJhM1ewHBh28QvI8kV+qSDgnOV5KYzd8trapLWeiPnXe22lVAPgmNb6E8zj5a3s3oOKuROYpfX/t3P3rlFFQRiHf0ZEi0AEEYXgR/eCWIq1sbCxSCEIGhXR0iL+AQqJwUIEIZW1gogSO0GMWFjEiBhNNAROZSfBwkZsoiEWMzGbuC5Rs7p38z7lcrkfe/ecnTsz55Y9pZS9pZRdwHvgE3BMUoekHWSwBhRgu6LJFUmbstxWCQ3GRq0OljKOJ4lSCpKOrigdzBOlEYBeSVvyd3SIGBtVUXfurOMp0deCpI2Supp9Yv9YF/Axg5Yelmfcaz0GzmWGGUndi3O9VUsrBC4DRIligmgYXHRR0nSm/74Cj4iJZUrSGyKjMJzNd2eBu7ntOJECbGQQOCJpmgg0ZoHPJVYnXAJGc19PWCqZ1JXNjQN53DGil+JPdQK3JM3k8fflvgeBYUmviEm3kREi0Gi0cuIecIrlZa0+4LykKaIG3puf9wMX8gm3+/cup3myAfkq8CzP+UadzS4TgeUY0YS86Lqkd3n/nxMN1ceB6UzH7wduN/H018IJfs6uPAB2Ev05M0S55DXRgDxH/Klfy+9rkigPVsWvxkatL8DBvK+HgSv5+Wmix2WSKHf21WRP3xIlohfAUCnlQ1OvYm0NUH/uXKkf6MkxPMHf9eG1ojvAgby+Mywf6z+UUkaJhQbjue0I8XBjFbNhYaFVM6PNI2kzMF9K+ZZPoDcblEnMKkVSZ4lVdduAl0Sz+ez/Pq9Wo3X2zh+zdtGOa/xXYzdwX7F0dI54iZVZu3goaSuxTHbIQYuZtZN1mXExMzOzamqFHhczMzOzVXHgYmZmZpXhwMXMzMwqw4GLmZmZVYYDFzMzM6sMBy5mZmZWGd8BDso4bWmdRsUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Just to see the correlation\n",
    "plt.figure(figsize=(10,8))\n",
    "sns.heatmap(train_data.corr(method='pearson'), annot=True, cmap='YlGnBu', fmt='.2f', linewidths=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "_uuid": "652d3d496e465bf60d5aef33f79e6f180c082dc7"
   },
   "outputs": [],
   "source": [
    "feature_columns = []\n",
    "\n",
    "# numeric cols\n",
    "for header in ['Age', 'Fare']:\n",
    "    feature_columns.append(tf.feature_column.numeric_column(header))\n",
    "\n",
    "# bucketized cols\n",
    "age = tf.feature_column.numeric_column(\"Age\")\n",
    "age_buckets = tf.feature_column.bucketized_column(age, boundaries=[5, 10, 20, 30, 40, 50, 60, 70, 80])\n",
    "feature_columns.append(age_buckets)\n",
    "\n",
    "# indicator cols\n",
    "categorical_cols = [\"Sex\", \"Embarked\", \"Pclass\", \"SibSp\", \"Parch\"]\n",
    "for col in categorical_cols:\n",
    "    train_data[col] = train_data[col].apply(str)\n",
    "    test_data[col] = test_data[col].apply(str)\n",
    "    cat_column_with_vocab = tf.feature_column.categorical_column_with_vocabulary_list(col, list(train_data[col].value_counts().index.values))\n",
    "    one_hot = tf.feature_column.indicator_column(cat_column_with_vocab)\n",
    "    feature_columns.append(one_hot)\n",
    "\n",
    "\n",
    "# embedding cols\n",
    "ticket = tf.feature_column.categorical_column_with_hash_bucket(\"Ticket\", hash_bucket_size=1000)\n",
    "ticket_embedding = tf.feature_column.embedding_column(ticket, dimension=8)\n",
    "feature_columns.append(ticket_embedding)\n",
    "\n",
    "# crossed cols\n",
    "p_class = tf.feature_column.categorical_column_with_vocabulary_list(\"Pclass\", list(train_data[\"Pclass\"].value_counts().index.values))\n",
    "parch = tf.feature_column.categorical_column_with_vocabulary_list(\"Parch\", list(train_data[\"Parch\"].value_counts().index.values))\n",
    "pclass_parch_crossed = tf.feature_column.crossed_column([p_class, parch], hash_bucket_size=1000)\n",
    "pclass_parch_crossed = tf.feature_column.indicator_column(pclass_parch_crossed)\n",
    "feature_columns.append(pclass_parch_crossed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_uuid": "869e25aea4077c3327535181afa45dd68e91a21e"
   },
   "outputs": [],
   "source": [
    "# A utility method to create a tf.data dataset from a Pandas Dataframe\n",
    "def df_to_dataset(dataframe, testing=False, batch_size=32):\n",
    "    dataframe = dataframe.copy()\n",
    "    if not testing:\n",
    "        labels = dataframe.pop('Survived')\n",
    "        ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n",
    "        ds = ds.shuffle(buffer_size=len(dataframe))\n",
    "    else:\n",
    "        ds = tf.data.Dataset.from_tensor_slices(dict(dataframe))\n",
    "    ds = ds.batch(batch_size)\n",
    "    ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n",
    "    return ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "_uuid": "81002699b6013bc68b4ec8b6f236d2ac372df3f0"
   },
   "outputs": [],
   "source": [
    "train_data, val_data = train_test_split(train_data, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(569, 11)\n",
      "(143, 11)\n",
      "(418, 10)\n"
     ]
    }
   ],
   "source": [
    "print(train_data.shape)\n",
    "print(val_data.shape)\n",
    "print(test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "_uuid": "e1daff759cc6bd037835cda13c7ad1845257f77d"
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "train_ds = df_to_dataset(train_data, batch_size=batch_size)\n",
    "val_ds = df_to_dataset(val_data, batch_size=batch_size)\n",
    "test_ds = df_to_dataset(test_data, testing=True, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "bf4ed1dccc9630b9f75050016c4d5d4d450925c5"
   },
   "source": [
    "### Setup Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "_uuid": "f72637b62bcdaaaab92fa467bbdedf3ea0f69f7c"
   },
   "outputs": [],
   "source": [
    "num_units_list = [128, 256]\n",
    "dropout_rate_list = [0.2, 0.5] \n",
    "optimizer_list = ['adam', 'sgd'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "_uuid": "365d776a86a163d36eba9eeedf318be60a46ad5d"
   },
   "outputs": [],
   "source": [
    "# Utility method to create summary for tensorboard\n",
    "def create_experiment_summary(num_units_list, dropout_rate_list, optimizer_list):\n",
    "    num_units_list_val = struct_pb2.ListValue()\n",
    "    num_units_list_val.extend(num_units_list)\n",
    "    dropout_rate_list_val = struct_pb2.ListValue()\n",
    "    dropout_rate_list_val.extend(dropout_rate_list)\n",
    "    optimizer_list_val = struct_pb2.ListValue()\n",
    "    optimizer_list_val.extend(optimizer_list)\n",
    "    return hparams_summary.experiment_pb(hparam_infos=[api_pb2.HParamInfo(name='num_units',                              # The parameters being tracked\n",
    "                                                                          display_name='Number of units',\n",
    "                                                                          type=api_pb2.DATA_TYPE_FLOAT64,\n",
    "                                                                          domain_discrete=num_units_list_val),\n",
    "                                                       api_pb2.HParamInfo(name='dropout_rate',\n",
    "                                                                          display_name='Dropout rate',\n",
    "                                                                          type=api_pb2.DATA_TYPE_FLOAT64,\n",
    "                                                                          domain_discrete=dropout_rate_list_val),\n",
    "                                                       api_pb2.HParamInfo(name='optimizer',\n",
    "                                                                          display_name='Optimizer',\n",
    "                                                                          type=api_pb2.DATA_TYPE_STRING,\n",
    "                                                                          domain_discrete=optimizer_list_val)], \n",
    "                                         metric_infos=[api_pb2.MetricInfo(name=api_pb2.MetricName(tag='accuracy'),        # The metrics being tracked\n",
    "                                                                          display_name='Accuracy'),\n",
    "                                                      ])\n",
    "\n",
    "exp_summary = create_experiment_summary(num_units_list, dropout_rate_list, optimizer_list)\n",
    "root_logdir_writer = tf.summary.create_file_writer(\"logs/hparam_tuning\")\n",
    "\n",
    "with root_logdir_writer.as_default():\n",
    "    #tf.summary.import_event(tf.compat.v1.Event(summary=exp_summary).SerializeToString()) #https://github.com/tensorflow/tensorboard/pull/2110\n",
    "    tf.summary.experimental.write_raw_pb(tf.compat.v1.Event(summary=exp_summary).SerializeToString(), step=tf.compat.v1.Event(summary=exp_summary).step)\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "_uuid": "00c52b5e1da3208d7a718602f9783ba045f1742b"
   },
   "outputs": [],
   "source": [
    "# Model compiler\n",
    "def train_test_model(hparams):\n",
    "    tf.keras.backend.set_floatx('float64')\n",
    "    model = tf.keras.models.Sequential([tf.keras.layers.DenseFeatures(feature_columns),\n",
    "                                        tf.keras.layers.Dense(hparams['num_units'], activation='relu'),\n",
    "                                        tf.keras.layers.Dropout(hparams['dropout_rate']),\n",
    "                                        tf.keras.layers.Dense(hparams['num_units'], activation='relu'),\n",
    "                                        tf.keras.layers.Dense(1, activation='sigmoid')])\n",
    "    \n",
    "    model.compile(optimizer=hparams['optimizer'],\n",
    "                  loss='binary_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    model.fit(train_ds,\n",
    "              validation_data=val_ds,\n",
    "              epochs=50,\n",
    "              use_multiprocessing=True,\n",
    "              verbose=0)\n",
    "    \n",
    "    _, accuracy = model.evaluate(val_ds)\n",
    "    return model, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "_uuid": "ce40f4ef18a980e04f4647e25a97c39345fe3f64"
   },
   "outputs": [],
   "source": [
    "# Model runner\n",
    "def run(run_dir, hparams):\n",
    "    writer = tf.summary.create_file_writer(run_dir)\n",
    "    summary_start = hparams_summary.session_start_pb(hparams=hparams)\n",
    "    with writer.as_default():\n",
    "        model, accuracy = train_test_model(hparams)\n",
    "        summary_end = hparams_summary.session_end_pb(api_pb2.STATUS_SUCCESS)\n",
    "\n",
    "    tf.summary.scalar('accuracy', accuracy, step=1, description=\"The accuracy\")\n",
    "    #tf.summary.import_event(tf.compat.v1.Event(summary=summary_start).SerializeToString())\n",
    "    #tf.summary.import_event(tf.compat.v1.Event(summary=summary_end).SerializeToString())\n",
    "    tf.summary.experimental.write_raw_pb(tf.compat.v1.Event(summary=summary_start).SerializeToString(), step=tf.compat.v1.Event(summary=exp_summary).step)\n",
    "    tf.summary.experimental.write_raw_pb(tf.compat.v1.Event(summary=summary_end).SerializeToString(), step=tf.compat.v1.Event(summary=exp_summary).step)\n",
    "    return model, accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "  Level | Level for Humans | Level Description                  \n",
    " -------|------------------|------------------------------------ \n",
    "  0     | DEBUG            | [Default] Print all messages       \n",
    "  1     | INFO             | Filter out INFO messages           \n",
    "  2     | WARNING          | Filter out INFO & WARNING messages \n",
    "  3     | ERROR            | Filter out all messages      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "_uuid": "28941b640efbd4b5dd7973f072f2d4dc20ae2a8c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Running training session 1\n",
      "{'num_units': 128, 'dropout_rate': 0.2, 'optimizer': 'adam'}\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.9213 - accuracy: 0.7832\n",
      "0.7832167832167832\n",
      "--- Running training session 2\n",
      "{'num_units': 128, 'dropout_rate': 0.2, 'optimizer': 'sgd'}\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.5248 - accuracy: 0.7552\n",
      "0.7552447552447552\n",
      "--- Running training session 3\n",
      "{'num_units': 128, 'dropout_rate': 0.5, 'optimizer': 'adam'}\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.9234 - accuracy: 0.7273\n",
      "0.7272727272727273\n",
      "--- Running training session 4\n",
      "{'num_units': 128, 'dropout_rate': 0.5, 'optimizer': 'sgd'}\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.5527 - accuracy: 0.6993\n",
      "0.6993006993006993\n",
      "--- Running training session 5\n",
      "{'num_units': 256, 'dropout_rate': 0.2, 'optimizer': 'adam'}\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 1.1534 - accuracy: 0.7203\n",
      "0.7202797202797203\n",
      "--- Running training session 6\n",
      "{'num_units': 256, 'dropout_rate': 0.2, 'optimizer': 'sgd'}\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.5247 - accuracy: 0.7273\n",
      "0.7272727272727273\n",
      "--- Running training session 7\n",
      "{'num_units': 256, 'dropout_rate': 0.5, 'optimizer': 'adam'}\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 1.1827 - accuracy: 0.6993\n",
      "0.6993006993006993\n",
      "--- Running training session 8\n",
      "{'num_units': 256, 'dropout_rate': 0.5, 'optimizer': 'sgd'}\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.5480 - accuracy: 0.7413\n",
      "0.7412587412587412\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "model_dict = {}\n",
    "session_num = 0\n",
    "for num_units in num_units_list:\n",
    "    for dropout_rate in dropout_rate_list:\n",
    "        for optimizer in optimizer_list:\n",
    "            hparams = {'num_units': num_units, 'dropout_rate': dropout_rate, 'optimizer': optimizer}\n",
    "            print(f'--- Running training session {session_num + 1}')\n",
    "            print(hparams)\n",
    "            run_name = f\"run-{session_num}\"\n",
    "            model, accuracy = run(\"logs/hparam_tuning/\" + run_name, hparams)\n",
    "            print(accuracy)\n",
    "            model_dict[accuracy] = model\n",
    "            session_num += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "_uuid": "91a3242fdeccecb9d49e00c2c0462bb81736165f"
   },
   "outputs": [],
   "source": [
    "best_model = model_dict[max(list(model_dict.keys()))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "_uuid": "6bc6e36e0a4064ebd24c1565e1388e700020bfe2"
   },
   "outputs": [],
   "source": [
    "predictions = best_model.predict(test_ds)\n",
    "predictions = np.argmax(predictions, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "_uuid": "861ce10c0b7becb306c1227f03ce236cf10fd780"
   },
   "outputs": [],
   "source": [
    "predictions_dataframe = test_data[[\"PassengerId\"]]\n",
    "predictions_dataframe[\"Survived\"] = predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "_uuid": "0986b2e992c217af9c3d9c45a1ced4502e963afe"
   },
   "outputs": [],
   "source": [
    "#predictions_dataframe.to_csv(\"../../../data/gender_submission.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "_uuid": "0f3149b86a86d81e8c7f1b71d5963f46b2632ea2"
   },
   "outputs": [],
   "source": [
    "#best_model.save('../../../data/best_model.h5')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}