keypoint_classification_EN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "igMyGnjE9hEp"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-06 20:31:03.297565: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-04-06 20:31:03.538198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2023-04-06 20:31:03.538228: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "2023-04-06 20:31:04.679994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-06 20:31:04.680072: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-06 20:31:04.680079: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "RANDOM_SEED = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t2HDvhIu9hEr"
   },
   "source": [
    "# Specify each path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "id": "9NvZP2Zn9hEy"
   },
   "outputs": [],
   "source": [
    "dataset = 'model/keypoint_classifier/keypoint.csv'\n",
    "model_save_path = 'model/keypoint_classifier/keypoint_classifier.hdf5'\n",
    "tflite_save_path = 'model/keypoint_classifier/keypoint_classifier.tflite'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "s5oMH7x19hEz"
   },
   "source": [
    "# Set number of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "id": "du4kodXL9hEz"
   },
   "outputs": [],
   "source": [
    "NUM_CLASSES = 7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XjnL0uso9hEz"
   },
   "source": [
    "# Dataset reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "id": "QT5ZqtEz9hE0"
   },
   "outputs": [],
   "source": [
    "X_dataset = np.loadtxt(dataset, delimiter=',', dtype='float32', usecols=list(range(1, (21 * 2) + 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "id": "QmoKFsp49hE0"
   },
   "outputs": [],
   "source": [
    "y_dataset = np.loadtxt(dataset, delimiter=',', dtype='int32', usecols=(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "id": "xQU7JTZ_9hE0"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_dataset, y_dataset, train_size=0.75, random_state=RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mxK_lETT9hE0"
   },
   "source": [
    "# Model building MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "id": "vHBmUf1t9hE1"
   },
   "outputs": [],
   "source": [
    "model_1 = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Input((21 * 2, )),\n",
    "    tf.keras.layers.Dropout(0.2),\n",
    "    tf.keras.layers.Dense(20, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.4),\n",
    "    tf.keras.layers.Dense(10, activation='relu'),\n",
    "    tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ypqky9tc9hE1",
    "outputId": "5db082bb-30e3-4110-bf63-a1ee777ecd46",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_33\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dropout_30 (Dropout)        (None, 42)                0         \n",
      "                                                                 \n",
      " dense_81 (Dense)            (None, 20)                860       \n",
      "                                                                 \n",
      " dropout_31 (Dropout)        (None, 20)                0         \n",
      "                                                                 \n",
      " dense_82 (Dense)            (None, 10)                210       \n",
      "                                                                 \n",
      " dense_83 (Dense)            (None, 7)                 77        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,147\n",
      "Trainable params: 1,147\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_1.summary()  # tf.keras.utils.plot_model(model, show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "id": "c3Dac0M_9hE2"
   },
   "outputs": [],
   "source": [
    "# Model compilation\n",
    "model_1.compile(\n",
    "    optimizer='adam',\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model building CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2 = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Conv1D(32, 3, activation='relu', input_shape=(42, 1)),\n",
    "    tf.keras.layers.MaxPooling1D(2),\n",
    "    tf.keras.layers.Conv1D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling1D(2),\n",
    "    tf.keras.layers.Conv1D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(64, activation='relu'),\n",
    "    tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_27\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv1d_51 (Conv1D)          (None, 40, 32)            128       \n",
      "                                                                 \n",
      " max_pooling1d_34 (MaxPoolin  (None, 20, 32)           0         \n",
      " g1D)                                                            \n",
      "                                                                 \n",
      " conv1d_52 (Conv1D)          (None, 18, 64)            6208      \n",
      "                                                                 \n",
      " max_pooling1d_35 (MaxPoolin  (None, 9, 64)            0         \n",
      " g1D)                                                            \n",
      "                                                                 \n",
      " conv1d_53 (Conv1D)          (None, 7, 64)             12352     \n",
      "                                                                 \n",
      " flatten_17 (Flatten)        (None, 448)               0         \n",
      "                                                                 \n",
      " dense_66 (Dense)            (None, 64)                28736     \n",
      "                                                                 \n",
      " dense_67 (Dense)            (None, 7)                 455       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 47,879\n",
      "Trainable params: 47,879\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_2.summary()  # tf.keras.utils.plot_model(model, show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model compilation\n",
    "model_2.compile(\n",
    "    optimizer='adam',\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model building LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3 = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.LSTM(42, input_shape=(42, 1)),\n",
    "    tf.keras.layers.Dropout(0.5),\n",
    "    tf.keras.layers.Dense(20, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.4),\n",
    "    tf.keras.layers.Dense(10, activation='relu'),\n",
    "    tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_31\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " lstm_2 (LSTM)               (None, 42)                7392      \n",
      "                                                                 \n",
      " dropout_26 (Dropout)        (None, 42)                0         \n",
      "                                                                 \n",
      " dense_75 (Dense)            (None, 20)                860       \n",
      "                                                                 \n",
      " dropout_27 (Dropout)        (None, 20)                0         \n",
      "                                                                 \n",
      " dense_76 (Dense)            (None, 10)                210       \n",
      "                                                                 \n",
      " dense_77 (Dense)            (None, 7)                 77        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 8,539\n",
      "Trainable params: 8,539\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_3.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model compilation\n",
    "model_3.compile(\n",
    "    optimizer='adam',\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model building MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4 = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.LSTM(42, input_shape=(42, 1)),\n",
    "    tf.keras.layers.Dropout(0.5),\n",
    "    tf.keras.layers.Dense(20, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.4),\n",
    "    tf.keras.layers.Dense(10, activation='relu'),\n",
    "    tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CallBack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model checkpoint callback\n",
    "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    model_save_path, verbose=1, save_weights_only=False)\n",
    "# Callback for early stopping\n",
    "es_callback = tf.keras.callbacks.EarlyStopping(patience=20, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7XI0j1Iu9hE2"
   },
   "source": [
    "# Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WirBl-JE9hE3",
    "outputId": "71b30ca2-8294-4d9d-8aa2-800d90d399de",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5890 - accuracy: 0.8203\n",
      "Epoch 1: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 3ms/step - loss: 0.6420 - accuracy: 0.7654 - val_loss: 0.3203 - val_accuracy: 0.9014\n",
      "Epoch 2/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6510 - accuracy: 0.7812\n",
      "Epoch 2: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6485 - accuracy: 0.7586 - val_loss: 0.3430 - val_accuracy: 0.8928\n",
      "Epoch 3/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6409 - accuracy: 0.7578\n",
      "Epoch 3: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6257 - accuracy: 0.7683 - val_loss: 0.3246 - val_accuracy: 0.9058\n",
      "Epoch 4/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6073 - accuracy: 0.8047\n",
      "Epoch 4: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6385 - accuracy: 0.7577 - val_loss: 0.3233 - val_accuracy: 0.9051\n",
      "Epoch 5/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5670 - accuracy: 0.7969\n",
      "Epoch 5: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6448 - accuracy: 0.7608 - val_loss: 0.3342 - val_accuracy: 0.9000\n",
      "Epoch 6/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6016 - accuracy: 0.7578\n",
      "Epoch 6: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 3ms/step - loss: 0.6266 - accuracy: 0.7664 - val_loss: 0.3291 - val_accuracy: 0.8971\n",
      "Epoch 7/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5875 - accuracy: 0.7734\n",
      "Epoch 7: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 3ms/step - loss: 0.6617 - accuracy: 0.7548 - val_loss: 0.3310 - val_accuracy: 0.9043\n",
      "Epoch 8/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.7059 - accuracy: 0.7578\n",
      "Epoch 8: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6563 - accuracy: 0.7538 - val_loss: 0.3379 - val_accuracy: 0.8978\n",
      "Epoch 9/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6834 - accuracy: 0.7500\n",
      "Epoch 9: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6259 - accuracy: 0.7698 - val_loss: 0.3260 - val_accuracy: 0.9043\n",
      "Epoch 10/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6098 - accuracy: 0.7500\n",
      "Epoch 10: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6263 - accuracy: 0.7637 - val_loss: 0.3417 - val_accuracy: 0.8884\n",
      "Epoch 11/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5546 - accuracy: 0.7734\n",
      "Epoch 11: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6128 - accuracy: 0.7777 - val_loss: 0.3163 - val_accuracy: 0.9022\n",
      "Epoch 12/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.7185 - accuracy: 0.7109\n",
      "Epoch 12: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6453 - accuracy: 0.7618 - val_loss: 0.3340 - val_accuracy: 0.8949\n",
      "Epoch 13/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.7165 - accuracy: 0.7500\n",
      "Epoch 13: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6326 - accuracy: 0.7736 - val_loss: 0.3236 - val_accuracy: 0.9000\n",
      "Epoch 14/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5443 - accuracy: 0.7422\n",
      "Epoch 14: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6282 - accuracy: 0.7596 - val_loss: 0.3331 - val_accuracy: 0.8942\n",
      "Epoch 15/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.7116 - accuracy: 0.7266\n",
      "Epoch 15: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6097 - accuracy: 0.7664 - val_loss: 0.3362 - val_accuracy: 0.8920\n",
      "Epoch 16/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5742 - accuracy: 0.7500\n",
      "Epoch 16: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6191 - accuracy: 0.7719 - val_loss: 0.3274 - val_accuracy: 0.8891\n",
      "Epoch 17/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.7401 - accuracy: 0.7734\n",
      "Epoch 17: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6365 - accuracy: 0.7681 - val_loss: 0.3344 - val_accuracy: 0.8884\n",
      "Epoch 18/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5784 - accuracy: 0.7891\n",
      "Epoch 18: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6343 - accuracy: 0.7647 - val_loss: 0.3243 - val_accuracy: 0.8993\n",
      "Epoch 19/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5949 - accuracy: 0.7969\n",
      "Epoch 19: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6269 - accuracy: 0.7669 - val_loss: 0.3236 - val_accuracy: 0.9007\n",
      "Epoch 20/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6469 - accuracy: 0.6953\n",
      "Epoch 20: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6258 - accuracy: 0.7627 - val_loss: 0.3328 - val_accuracy: 0.8942\n",
      "Epoch 21/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6626 - accuracy: 0.7578\n",
      "Epoch 21: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6220 - accuracy: 0.7620 - val_loss: 0.3341 - val_accuracy: 0.8949\n",
      "Epoch 22/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5651 - accuracy: 0.7578\n",
      "Epoch 22: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6348 - accuracy: 0.7623 - val_loss: 0.3407 - val_accuracy: 0.8855\n",
      "Epoch 23/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5853 - accuracy: 0.7734\n",
      "Epoch 23: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6318 - accuracy: 0.7647 - val_loss: 0.3356 - val_accuracy: 0.8957\n",
      "Epoch 24/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6186 - accuracy: 0.7656\n",
      "Epoch 24: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6149 - accuracy: 0.7714 - val_loss: 0.3347 - val_accuracy: 0.8906\n",
      "Epoch 25/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5384 - accuracy: 0.7812\n",
      "Epoch 25: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6293 - accuracy: 0.7681 - val_loss: 0.3337 - val_accuracy: 0.9007\n",
      "Epoch 26/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6601 - accuracy: 0.7266\n",
      "Epoch 26: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6216 - accuracy: 0.7724 - val_loss: 0.3297 - val_accuracy: 0.8993\n",
      "Epoch 27/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5602 - accuracy: 0.7734\n",
      "Epoch 27: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6266 - accuracy: 0.7647 - val_loss: 0.3297 - val_accuracy: 0.8986\n",
      "Epoch 28/10000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1/33 [..............................] - ETA: 0s - loss: 0.7138 - accuracy: 0.7656\n",
      "Epoch 28: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6419 - accuracy: 0.7681 - val_loss: 0.3402 - val_accuracy: 0.8848\n",
      "Epoch 29/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.6942 - accuracy: 0.7578\n",
      "Epoch 29: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6404 - accuracy: 0.7666 - val_loss: 0.3281 - val_accuracy: 0.8964\n",
      "Epoch 30/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5924 - accuracy: 0.7734\n",
      "Epoch 30: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6491 - accuracy: 0.7664 - val_loss: 0.3436 - val_accuracy: 0.8920\n",
      "Epoch 31/10000\n",
      " 1/33 [..............................] - ETA: 0s - loss: 0.5624 - accuracy: 0.7969\n",
      "Epoch 31: saving model to model/keypoint_classifier/keypoint_classifier.hdf5\n",
      "33/33 [==============================] - 0s 2ms/step - loss: 0.6248 - accuracy: 0.7659 - val_loss: 0.3291 - val_accuracy: 0.8971\n",
      "Epoch 31: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f7403704880>"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1.fit(\n",
    "    X_train,\n",
    "    y_train,\n",
    "    epochs=10000,\n",
    "    batch_size=128,\n",
    "    validation_data=(X_test, y_test),\n",
    "    callbacks=[cp_callback, es_callback],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pxvb2Y299hE3",
    "outputId": "59eb3185-2e37-4b9e-bc9d-ab1b8ac29b7f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11/11 [==============================] - 0s 1ms/step - loss: 0.3291 - accuracy: 0.8971\n"
     ]
    }
   ],
   "source": [
    "# Model evaluation\n",
    "model = model_1\n",
    "val_loss, val_acc = model.evaluate(X_test, y_test, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "id": "RBkmDeUW9hE4"
   },
   "outputs": [],
   "source": [
    "# Loading the saved model\n",
    "model = tf.keras.models.load_model(model_save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tFz9Tb0I9hE4",
    "outputId": "1c3b3528-54ae-4ee2-ab04-77429211cbef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 36ms/step\n",
      "[3.6126648e-03 8.1917256e-01 1.7721476e-01 5.5073636e-18 6.9420900e-20\n",
      " 1.9496808e-23 2.4059578e-21]\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "# Inference test\n",
    "predict_result = model.predict(np.array([X_test[0]]))\n",
    "print(np.squeeze(predict_result))\n",
    "print(np.argmax(np.squeeze(predict_result)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "S3U4yNWx9hE4"
   },
   "source": [
    "# Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 582
    },
    "id": "AP1V6SCk9hE5",
    "outputId": "08e41a80-7a4a-4619-8125-ecc371368d19"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44/44 [==============================] - 0s 740us/step\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAH5CAYAAACWFaT0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABWxElEQVR4nO3deVxUZdsH8N+wjezGjvueouKKSOBOIu65lXtlmoaWkmaUiVtiLrnklunrUpJbau6mmCgJirigqCi44MIiomzKsMy8f/g4NQ2mo5w5c5jf9/mczxPnnLnnuhiU2+tejkylUqlAREREJGEmYgdARERE9LrYoSEiIiLJY4eGiIiIJI8dGiIiIpI8dmiIiIhI8tihISIiIsljh4aIiIgkjx0aIiIikjwzsQN4pijzutghiMKyUhuxQxCFTOwAROJgaSt2CKJ48CRX7BBIj+Rm5mKHIIr8xzf19l5C/s40d6olWNtCYoWGiIiIJM9gKjRERET0kpQlYkdgcFihISIiIsljhYaIiEhqVEqxIzA4rNAQERGR5LFCQ0REJDVKVmj+jR0aIiIiiVFxyEkLh5yIiIhI8lihISIikhoOOWlhhYaIiIgkjxUaIiIiqeEcGi2s0BAREZHksUJDREQkNXz0gRZWaIiIiEjyWKEhIiKSGs6h0cIKDREREUkeKzRERERSw31otLBDQ0REJDF89IE2DjkRERGR5LFCQ0REJDUcctLCCg0RERFJHis0REREUsM5NFpYoSEiIiLJY4WGiIhIavjoAy2s0BAREZHksUJDREQkNZxDo4UdGiIiIqnhsm0t5XLIadOOPXhn2Bh4v90H3m/3weBRE3A8OlZ9PfNBFr6cMQ/tegyCV6fe6P/BWBz6M0qjjeycXEye9h283+4Dn4B++CZsIR4/fqLvVMpcGz9v7NyxDik341BceBc9ewaIHZLgvvhiLKJP7EXWg0TcvXMe27atQb16tcUOq8x9GjwKB/7ciuQ7cUhI+gvrNi5F7To11dcrvmGP2XOn4K/T+3Ez7RziLh7Bt999DVs7GxGjFs6Y0cORdDUGeTnJOBG1G14tm4odkl6U97x9fVth67bVSEo+ifzHN9G9R2ete6Z8MwHJ108h88EV7NnzC2rXrqH/QEnvymWHxs3ZCRNGf4At//cDNq9ZglYtmmDclzOQdP0WACBk5nzcTLmDpd+FYvuGFfBv54vPp4bh8tUkdRuTp89F0o0U/LRoNpbNnYa4cxcxbe4SsVIqM9bWVoiPv4Rxn30tdih607ZNa6xYsR5+bXogsOtAmJuZY9/ecFhZWYodWpny8fXC2p/C0dX/XfTv/SHMzM2wecdqdZ5ubi5wdXfB9Clz0c6nBz77JAQd/Ntg4dJvRY687PXv3xPz54Vi5qzv4eXdBefjL2Hf3o1wdnYUOzRBGUPe1tZWuHDhMiZMmFrq9eDg0Rgz5gN8+unXaN+uN/IfP8HvuzZALpfrOVKBqZTCHRIlU6lUKrGDAICizOuCtv9Wl/74POgj9O0RAC//d/DNxLHo2aWT+rpv4ABMGPMh+vXsguSbKeg1+GNsWr0YjRrUAwBExZzGmIlTEbHjZ7iU4V8OlpXalFlbuiouvIs+/T7Erl0H9f7eMr2/49+cnByQeu8COnTsg6iok3p9bwdLW729l6PjG7h0PRq9Aocg5sTpUu/p0TsAy1bNQ033ZigpEW7VxIMnuYK1XZoTUbsRe/o8Phs/BQAgk8lw83osli1fi7nzluk1Fn0ylLzlZuZ6eZ/8xzfx7rujsGf3H+pzyddPYcnin7B48U8AADs7W9y4eRofj5qIbdt2Cx6PviguHhKsbXmjtwVrW0g6V2gyMzMxd+5cvPPOO/Dx8YGPjw/eeecdzJs3D/fv3xcixtdSUlKCfYeP4klBAZo2qg8AaNqoAQ5EHEN2Ti6USiX2HT6KwsJCtGruCQA4f/Ey7Gxt1J0ZAGjdshlMTGSIv3RFlDyo7Njb2wEAHj58JG4gArO1f9p5evQw+7n32NnZIjc3T9DOjL6Zm5ujeXNPRBw5rj6nUqkQcSQKrVu3EDEyYRlr3v9Uo0ZVuLm54M8//1Kfy8nJRWzsOXh7NxcxMgEolcIdEqXTpODY2FgEBATAysoK/v7+qFfv6S/89PR0LFmyBHPmzMHBgwfRsmXL/2xHoVBAoVBonDNRKMq0JHg1+QYGfxyMwsJCWFlaYvHsb1C7ZnUAwIKZX2Hi1DD4Bg6AmakpKlSQY9Hsb1CtSiUAQOaDh3CoaK/RnpmZKextbZGZ9bDMYiT9k8lkWDB/Ov766xQSEhLFDkcwMpkMs8K+wsnoOFy5fK3UexwcKmLCpDH4Zd0WPUcnLCcnB5iZmSEjPVPjfEbGfdR/s/zNnXrGWPP+J1dXZwBPc/6njIz7cPnfNSq/dKrQjBs3Dv3798ft27exbt06fPfdd/juu++wbt06pKSkoF+/fhg3btwL2wkLC4O9vb3G8d3ila+cRGlqVquC39YtQ/iqRRjQuxu+/nYBkm88nUOz9KcNyM3Lx+rFs7FpzRIMe68PJk4Nw9XkG2UaAxmeH5bMRsOGb2LwkE/EDkVQcxZMxZsN6uLjD4NLvW5ja42NW3/E1cRkzAtbqufoiOh1qVQlgh26WLFiBTw9PWFnZwc7Ozv4+Phg//796uvt27eHTCbTOEaPHq3RRkpKCrp16wYrKyu4uLhg0qRJKC4u1vl7olOF5vz581i3bh1kMu0ZEDKZDBMmTECzZs1e2E5ISAiCgzX/ojXJvatLKC9kbm6urrg0rF8XCVeu4petv+ODQf0Q/ttu7Px5JerUelqxqV+3Fs6cv4hff9uD0C/GwcnxDWQ90izTFxeXIDs3F04Ob5RpnKQ/ixfNQteu/ujYqQ/u3k0VOxzBzJ73Dd4OaI/eXYcg9V661nVrG2ts+m018vLy8cHgsa/0F4chy8zMQnFxMVxcnTTOu7g4Iy3d8IbFy4qx5v1P6f/L08XFGWlpf+fs4uKMC/GXxAqrXKtSpQrmzJmDunXrQqVSYf369ejVqxfOnj2Lhg0bAgBGjhyJGTNmqF9jZWWl/u+SkhJ069YNbm5uOHHiBFJTUzFs2DCYm5tj9uzZOsWiU4XGzc0Np06deu71U6dOwdXV9YXtyOVydW/u2SH0DHSlUoXCwiIU/G+oS2ai2SkzMTGB6n+zu5s0aoCc3DwkXPm7VH8y7hyUShU8PeoLGicJY/GiWejVqws6BwzAzZu3xQ5HMLPnfYOu3f3Rt8f7SLml/Y8EG1trbNmxBoWFRRj23idQKApFiFJYRUVFOHMmHh07+KnPyWQydOzgh5iYOBEjE5ax5v1PN2/eRlpaBtq3f0t9ztbWBl5eTXHy5BkRIxOAgaxy6tGjB7p27Yq6deuiXr16+Pbbb2FjY4OYmBj1PVZWVnBzc1MfdnZ26mt//PEHLl26hF9++QVNmzZFYGAgZs6ciWXLlqGwULe/n3Sq0EycOBGjRo1CXFwcOnXqpO68pKenIyIiAj/99BPmz5+vUwBCWLhiLdr4tIS7qwvyHz/G3j+OIvZsPH78fhZqVq+KalUqYcbcHzBx7Eewt7PFkePRiI49i2VzpwEAateoBr/WLTHtu8WYOmkcioqLMXvhCgT6tyvTFU5isLa2Qp1/7E1Ss0Y1NGnSEFlZD3H79j0RIxPOD0tm4733eqNP3w+Rm5unHmfPzs5FQUGByNGVnTkLpqJPv+4YPigIeXn5cHZ5+i/13JxcFBQo1J0ZS0tLfDJqEmxsbWBj+3QPmgeZWVBKeDLgvy1c/BPWrlmIuDPxiI09i0/HjYS1tSXWrd8sdmiCMoa8ra2tNPaVqVG9Kjw9PZCV9Qh37tzDsqX/hy8mj0NS8k3cunkb30z9HKmp6dj9j5VQ5YKAf15Lm+cql8tfWHgoKSnB1q1bkZ+fDx8fH/X5jRs34pdffoGbmxt69OiBb775Rl2liY6ORuPGjTWKIQEBARgzZgwSEhJeatTnGZ2XbW/evBkLFy5EXFycemWEqakpWrRogeDgYAwYMECX5tTKctn2N2ELcfL0Odx/kAVba2vUq1MTHw7uj7daPZ3lfuv2XSxcsRZn4hPw5MkTVK1SCe8P7KuxjDs7Jxfffr8cR6NOwsREBv/2vvhq/Jgy37tE38u227X1QcThbVrn12/YghEfTdBbHPpctl1UWPpw5ogRE7DhZ/1OiBVy2XZ6dukr8D4dE4LN4Tvwll8r7Ni7odR7WjbuhNspZTvs+0/6XrYNAJ+MeR+fB4+Bm5szzp9PwPgJU3Eq9qze49A3Q8hbyGXbbdq0xoGDm7TO//LzNnz88UQATzfW+/DDQbC3t0P0iViMH/8NkpKEnyOpz2XbBWd2Cdb2nF1nMH36dI1zoaGhmDZtWqn3X7hwAT4+PigoKICNjQ3Cw8PRtWtXAMCqVatQvXp1VKpUCfHx8Zg8eTJatWqF7du3AwBGjRqFW7du4eDBv7cPefz4MaytrbFv3z4EBga+dNyvvA9NUVERMjOfzqZ3cnKCufnr/QALvQ+NoRJzHxoxibkPjZj0uQ+NIRGjQ0Pi0dc+NIZGrx2auJ2CtS1rFKhThaawsBApKSnIzs7Gtm3bsHr1akRGRsLDw0Pr3iNHjqBTp05ISkpC7dq1y7RD88rPcjI3N4e7u/urvpyIiIgM0MsML/2ThYUF6tSpAwBo0aIFYmNjsXjxYvz4449a93p7ewOAukNT2tzc9PSnixnc3Nx0irtcPvqAiIioXFOWCHe8bmhKpVaF55lz584BgLog4uPjgwsXLiAjI0N9z6FDh2BnZ1dqhee/8GnbRERE9EpCQkIQGBiIatWqITc3F+Hh4Th69CgOHjyI5ORk9XwaR0dHxMfHY8KECWjbti08PZ/uzN+5c2d4eHhg6NChmDt3LtLS0jBlyhQEBQXpvPqZHRoiIiKpMZCHSGZkZGDYsGFITU2Fvb09PD09cfDgQbz99tu4ffs2Dh8+jEWLFiE/Px9Vq1ZF3759MWXKFPXrTU1NsWfPHowZMwY+Pj6wtrbG8OHDNfateVlG83BKQ8VJwcaFk4LJGHBSsPAKTm0VrO0KrfoL1raQWKEhIiKSmnK0b1RZYYeGiIhIagxkyMmQcJUTERERSR4rNERERFLDISctrNAQERGR5LFCQ0REJDWs0GhhhYaIiIgkjxUaIiIiiVGpXv8RBeUNKzREREQkeazQEBERSQ3n0Ghhh4aIiEhquLGeFg45ERERkeSxQkNERCQ1HHLSwgoNERERSR4rNERERFLDOTRaWKEhIiIiyWOFhoiISGo4h0YLKzREREQkeazQEBERSQ3n0Ghhh4aIiEhqOOSkhUNOREREJHms0BAREUkNKzRaDKZDY1ulvdghiCJ3/UdihyCKhuN2iR2CKFJyMsQOgfTI1MQ4i+CWZhZih0BGyGA6NERERPSSOClYi3H+84GIiIjKFVZoiIiIpIZzaLSwQkNERESSxwoNERGR1HAOjRZ2aIiIiKSGQ05aOOREREREkscKDRERkdRwyEkLKzREREQkeazQEBERSQ3n0GhhhYaIiIgkjxUaIiIiqWGFRgsrNERERCR5rNAQERFJjUoldgQGhx0aIiIiqeGQkxYOOREREZHksUJDREQkNazQaGGFhoiIiCSPFRoiIiKp4aMPtLBCQ0RERJLHCg0REZHUcA6NFlZoiIiISPJYoSEiIpIabqynhRUaIiIikjxWaIiIiKSGc2i0sENDREQkNezQaOGQExEREUmeUXZoEhP/QkFBitaxaNFMsUN7LVtir6H/8v3wnb0NvrO3YdjqQ4i6dk99fdvpJIxYGwHf2dvQdNom5Dwp1Grjs/Bj6PL9LrSauQX+83fi6+3RyMh5os80XtvgD/pjX+RmnL9xHOdvHMe2/evRrpOv+vqsBV/jz9hduHQ7GrFXjuDHnxeiVp0a4gUssDGjhyPpagzycpJxImo3vFo2FTskvTC2vKdMmQBFwW2NI/78n2KHVeZ83mqJjZtX4mLicWTmXEVgN3+N69bWVpgzfyriLx/D7fR4/HVqH97/8D2RohWQSincoYMVK1bA09MTdnZ2sLOzg4+PD/bv36++XlBQgKCgIDg6OsLGxgZ9+/ZFenq6RhspKSno1q0brKys4OLigkmTJqG4uFjnb4lRdmh8fXugevUW6qNr10EAgO3b94oc2etxtbPCp/5NEP5xAMJHdYZXTVeM/zUKSRnZAICCohL41nHHiDYez22jZU1XzO3/FnaO64b5A/xwOysPE7dE6SuFMpF6Lx1zZ/6AXp0Go7f/YEQfP4Uff16Ium/WAgBcPH8ZX3w6DW+/1QfvD/gEMpkMG7Yth4lJ+fvj0L9/T8yfF4qZs76Hl3cXnI+/hH17N8LZ2VHs0ARlrHknJCSiWvXm6qNDxz5ih1TmrKytcPHiFXzx+YxSr8+cHYKO/m0wZuREvOUViB+Xr8ec+VPRJbCjniM1DlWqVMGcOXMQFxeH06dPo2PHjujVqxcSEhIAABMmTMDu3buxdetWREZG4t69e+jT5++fy5KSEnTr1g2FhYU4ceIE1q9fj3Xr1mHq1Kk6xyJTqQxj7VeFCtVEe+9580LRtWsnNGzYVu/v/XDtB4K233bOdkzo3ATvNK+tPhd7Ix0j1/+JY5P7wM7S4j9ff/TKXUzYdBynvhkAc9Oy+4XfcNyuMmvrZZy5dhRzpi3Clo07ta7V96iLfce2oH3LHki5eUfQOFJyMgRt/99ORO1G7Onz+Gz8FACATCbDzeuxWLZ8LebOW6bXWPTJUPI21WMnecqUCejZIwCtvLvo7T2fx9bCUi/vk5lzFUMHfoL9ew+rzx2P2YOd2/dhwdzl6nMRkdtx+PAxhM1cJHg8+vJ41QTB2rYatfC1Xu/g4IB58+ahX79+cHZ2Rnh4OPr16wcAuHLlCho0aIDo6Gi0bt0a+/fvR/fu3XHv3j24uroCAFauXInJkyfj/v37sLD4799R/1T+/kmqI3Nzcwwc+A7Wr98sdihlqkSpxIELt/CkqBieVZxeqY3sxwrsu3ATTao6lWlnRp9MTEzQ/Z0AWFpZ4kxsvNZ1S6sK6DeoJ1Ju3kHq3TQRIhSOubk5mjf3RMSR4+pzKpUKEUei0Lp1CxEjE5ax5g0AderUxI3rp3HlchTWrVuCqlUriR2S3sWePIsuXTvBzf3pL0e/Nt6oXacGjkZIq9IsJoVCgZycHI1DoVC88HUlJSXYtGkT8vPz4ePjg7i4OBQVFcHf/+9hwfr166NatWqIjo4GAERHR6Nx48bqzgwABAQEICcnR13leVllvsrp9u3bCA0Nxf/93/899x6FQqH1zVGpVJDJZGUdzgv17BmAihXt8PPP2/T+3kK4lv4Iw1YfRmFxCSwtzPD9u36o7WKvUxuLDp3DplPXUFBUAs8qjlgySP+Vq9f1ZoM62LZ/PeQVLPA4/wnGDP8cSVevq68P+aA/JoeOh7WNFZKv3cCwfmNQVKT7mK0hc3JygJmZGTLSMzXOZ2TcR/03az/nVdJnrHnHnjqLj0YG4+rVZLi7ueLrr8cjIuI3NG/uj7y8fLHD05svJ83A90tm4WLicRQVFUGpVGHCp1MQfeK02KGVLQFXOYWFhWH69Oka50JDQzFt2rRS779w4QJ8fHxQUFAAGxsb7NixAx4eHjh37hwsLCxQsWJFjftdXV2Rlvb0H5BpaWkanZln159d00WZ/7M7KysL69ev/897wsLCYG9vr3GUlOSUdSgv5f3338XBg0eRmpr+4psloIajLTaPDsDPI9/GAK86mLrzJJL/N4fmZQ1/qwE2fxyAFUPbw0Qmw5QdMTCQkcmXdj3pJrp3eA99AoZh49qtmLd0BurUq6W+/vu2/ejRcSDe7TECN5JT8MOa72Ahf/nSJpGhOfjHUWzfvhcXL17BocOR6NV7OCra26Ffv+5ih6ZXIz8eipZeTTB4wMfo1LYPpn49B3PnT0Xb9m+JHZpkhISEIDs7W+MICQl57v1vvvkmzp07h5MnT2LMmDEYPnw4Ll26pMeIn9K5QrNr13/Pfbh+/fp/XgeefrOCg4M1zjk7N9Q1lNdWrVpldOzoh3ffHaX39xaKuZkpqjnaAgA8Kjkg4W4Wwk9exTc9vF66jTes5XjDWo7qTnao5WSHgIW7EH/nAZpUfbWhKzEUFRXj1o3bAJ5OAvZs1hDvfzwQUz7/FgCQm5uH3Nw83LyegnOn43E26RgCunXE7u0HxAy7TGVmZqG4uBgurpqfm4uLM9LS74sUlfCMNe9/y87OwbVrN1C7dg2xQ9GbChXk+Do0GMMHj8Whg0cBAJcSEtG4cQMEffohjh09IW6AZUnH1Ui6kMvlkMvlL32/hYUF6tSpAwBo0aIFYmNjsXjxYrz77rsoLCzEo0ePNKo06enpcHNzAwC4ubnh1KlTGu09WwX17J6XpXOHpnfv3pDJZP/5L/YXDR2V9s0SY7hp2LAByMh4gP37j+j9vfVFqVKhsLjktV4P4LXaMAQyE9lzJ5fJZDLIZICFhbmeoxJWUVERzpyJR8cOfti16yCAp7l27OCH5SvWihydcIw173+ztrZCrVrVER7+m9ih6I2ZuRksLCyg/NdwTElJSflbxag03Kq5UqmEQqFAixYtYG5ujoiICPTt2xcAkJiYiJSUFPj4+AAAfHx88O233yIjIwMuLi4AgEOHDsHOzg4eHs9fkVsanTs07u7uWL58OXr16lXq9XPnzqFFC8OfeCeTyTBsWH/88ss2lJRI+5f1M0sOn4dvHXe42VvhcWEx9l+4hdM3M7B8aHsAQGbuE2TmFeB2Vh4AICnjEawszOFubwV7Kzku3HmAhLsP0LSaM+wsLXAnKw/L/ryAqm/YSKo6M2nKOByN+Av37qTCxsYaPfsGorVvS7zf/xNUrV4Z3XsH4PjRaGRlPoRbJVeM/uwDFBQocPRw+Zs0uHDxT1i7ZiHizsQjNvYsPh03EtbWllhXzibB/5sx5j0nbAr27juMlJQ7cHd3xdRvglFSUoLNW34XO7QyZW1thZq1qqu/rl6jCho1boCHDx/h7p1U/HX8JKbN/AIFTwpw+/Y9vOXrhQEDe2PqV2EiRl1+hYSEIDAwENWqVUNubi7Cw8Nx9OhRHDx4EPb29hgxYgSCg4Ph4OAAOzs7jBs3Dj4+PmjdujUAoHPnzvDw8MDQoUMxd+5cpKWlYcqUKQgKCtKpSgS8QoemRYsWiIuLe26H5kXVG0PRqZMfqlWrUq5WN2XlF2DKjhhk5hXARm6Oeq4VsXxoe/jUflq223o6CT9G/j1r/MO1TytT03u1Qq9mtVDB3BQRl+9gxdGLeFJYDCdbS/jWccdH/T1gYWYqSk6vwtHJAQuWzYSzqxNyc/KQeOka3u//CaIiT8LFzRlerZvhg48Hwa6iHTLvP0Bs9Bn06/o+HmQ+FDv0Mrd16y44Ozlg2tSJcHNzxvnzCejWfQgyMjJf/GIJM8a8K1d2x4b1S+HoWBH372fhxIlYtG3XC5mZWWKHVqaaNmuE3/f9ov56VthXAIBfN27HuDFfYuQHEzBl2udYuXoBKr5hjzu372H2jIVYu+ZXsUIWhoE8+iAjIwPDhg1Damoq7O3t4enpiYMHD+Ltt98GACxcuBAmJibo27cvFAoFAgICsHz530vqTU1NsWfPHowZMwY+Pj6wtrbG8OHDMWNG6fsM/Red96E5fvw48vPz0aVL6Xsd5Ofn4/Tp02jXrp1OgYi5D42YhN6HxlDpex8aQ6HvfWhIXPrch8aQ6GsfGkOj131ofvhEsLatxi1/8U0GSOcKTZs2bf7zurW1tc6dGSIiItKBgVRoDIlx/vOBiIiIypUy31iPiIiIBCaBuar6xgoNERERSR4rNERERFLDOTRa2KEhIiKSGgPeWE8sHHIiIiIiyWOFhoiISGoEfJaTVLFCQ0RERJLHCg0REZHUcA6NFlZoiIiISPJYoSEiIpIYFZdta2GFhoiIiCSPFRoiIiKp4RwaLezQEBERSQ2XbWvhkBMRERFJHis0REREUsMhJy2s0BAREZHksUJDREQkNVy2rYUVGiIiIpI8VmiIiIikhnNotLBCQ0RERJLHCg0REZHUcB8aLezQEBERSQ2HnLRwyImIiIgkjxUaIiIiieHTtrWxQkNERESSZzAVGqWRTnCyG75a7BBEkfPTULFDEIXtyJ/FDoH0yNbCUuwQRPGoIF/sEMo/zqHRwgoNERERSZ7BVGiIiIjoJbFCo4UVGiIiIpI8VmiIiIikxkjnnf4XdmiIiIikhkNOWjjkRERERJLHCg0REZHEqFih0cIKDREREUkeKzRERERSwwqNFlZoiIiISPJYoSEiIpIaPpxSCys0REREJHms0BAREUkN59BoYYeGiIhIatih0cIhJyIiIpI8VmiIiIgkRqVihebfWKEhIiIiyWOFhoiISGo4h0YLKzREREQkeazQEBERSQ0rNFpYoSEiIiLJY4WGiIhIYlSs0GhhhYaIiEhqlCrhDh2EhYXBy8sLtra2cHFxQe/evZGYmKhxT/v27SGTyTSO0aNHa9yTkpKCbt26wcrKCi4uLpg0aRKKi4t1ioUVGiIiInolkZGRCAoKgpeXF4qLi/HVV1+hc+fOuHTpEqytrdX3jRw5EjNmzFB/bWVlpf7vkpISdOvWDW5ubjhx4gRSU1MxbNgwmJubY/bs2S8dCzs0REREUiPgw7YVCgUUCoXGOblcDrlcrnXvgQMHNL5et24dXFxcEBcXh7Zt26rPW1lZwc3NrdT3++OPP3Dp0iUcPnwYrq6uaNq0KWbOnInJkydj2rRpsLCweKm4OeREREREamFhYbC3t9c4wsLCXuq12dnZAAAHBweN8xs3boSTkxMaNWqEkJAQPH78WH0tOjoajRs3hqurq/pcQEAAcnJykJCQ8NJxs0JDREQkMUJOCg4JCUFwcLDGudKqM/+mVCoxfvx4+Pr6olGjRurzgwYNQvXq1VGpUiXEx8dj8uTJSExMxPbt2wEAaWlpGp0ZAOqv09LSXjpuo6zQjBo1FHGnDyHz/mVk3r+MY5G/IyCgg9hhCe6LL8Yi+sReZD1IxN0757Ft2xrUq1db7LBe25a46+j/UwR85++G7/zdGLb+KKKS//5DoCguwewD59Bu4R74zNuFz387iQd5BVrt/B5/C/1/ikCr735Hh0V7MfvAOT1mIZwxo4cj6WoM8nKScSJqN7xaNhU7JL0o73n7vNUSGzevxMXE48jMuYrAbv4a152dHfHDijm4mHgcKWnnsXn7atSqXV2kaIVX3j9vfZLL5bCzs9M4XqZDExQUhIsXL2LTpk0a50eNGoWAgAA0btwYgwcPxoYNG7Bjxw4kJyeXadxG2aG5ezcVX08JQ2ufrvB5qyuOHv0Lv21bA48G9cQOTVBt27TGihXr4demBwK7DoS5mTn27Q2HlZWl2KG9Flc7S3zaoSHCP+yA8A/aw6u6M8ZvjUHS/RwAwPxDF3AsKQ3z3vHGmiFtcD/vCYK3n9Ro4+eT17D06CV88FY9/DaqE34c5Ie3armW8m7S0r9/T8yfF4qZs76Hl3cXnI+/hH17N8LZ2VHs0ARlDHlbWVvh4sUr+OLzGaVe3/DrctSoURVDB36Cjn69cSflHn77fZ3k/7yXxhg+by0GssrpmbFjx2LPnj34888/UaVKlf+819vbGwCQlJQEAHBzc0N6errGPc++ft68m9LIVAbyyE4L+X9/A4SWlnoRX4bMwrp1m158cxkS89vv5OSA1HsX0KFjH0RFnXzxC8pQzk9DBW2/7fd7MKFjI/jXr4wOi/YirJcX3m5QGQBwIzMX76w6jA3D28GzsgNynhSi8w8HsLh/a3jXdBE0LtuRPwva/r+diNqN2NPn8dn4KQAAmUyGm9djsWz5Wsydt0yvseiToeRdsYL1i28qA5k5VzF04CfYv/cwAKB2nRo4eeYP+LbqisQrT39pyGQyXEo6gW+nf49fNmwVNJ5HBfmCtv9vhvJ5Fxfe1dt7PRoo3KhCxV//fOl7VSoVxo0bhx07duDo0aOoW7fuC1/z119/wc/PD+fPn4enpyf279+P7t27IzU1FS4uT/8OXrVqFSZNmoSMjIyXqg4BRlqh+ScTExMM6N8T1taWOBkTJ3Y4emVvbwcAePjwkbiBlKESpQoHEu7gSVEJPCs74HLaIxQrVfCu6ay+p6aTLdztLHH+bhYAIPpGBpQqFTLyCvDOj4fQ+Yf9mLT9FNJyHj/vbSTB3NwczZt7IuLIcfU5lUqFiCNRaN26hYiRCctY8/6nZ6tC/rlSRaVSoVBRCG+f8vU9MNrPWyngoYOgoCD88ssvCA8Ph62tLdLS0pCWloYnT54AAJKTkzFz5kzExcXh5s2b2LVrF4YNG4a2bdvC09MTANC5c2d4eHhg6NChOH/+PA4ePIgpU6YgKCjopTszwCt0aJ48eYKoqChcunRJ61pBQQE2bNjwwjYUCgVycnI0Dn1XKho1rI+sB4nIy72OpUvD0H/ASFy+ck2vMYhJJpNhwfzp+OuvU0hISHzxCwzctYxs+MzbhVbf/Y5ZB87h+77eqO1sh8z8ApibmsCuguayPwfrCup5NHcf5UOpUmHNX4mY9LYn5vfxRk5BIUaH/4WiEgHXRgrMyckBZmZmyEjP1DifkXEfbq7Oz3mV9Blr3v907ep13E65iymhn8O+oh3Mzc0xbvxIVK7iDle38vU94OctrhUrViA7Oxvt27eHu7u7+ti8eTOAp53rw4cPo3Pnzqhfvz4+//xz9O3bF7t371a3YWpqij179sDU1BQ+Pj4YMmQIhg0bprFvzcvQaZXT1atX0blzZ6SkpEAmk8HPzw+bNm2Cu7s7gKfLtT744AMMGzbsP9sJCwvD9OnTNc6ZmNjC1MxOp+BfR+LVZHi1CoCdnS369umGNasXwt+/n9F0an5YMhsNG76J9h3eETuUMlHD0RabR3REnqIIh6/cw9TdcVg9pM1LvVapAoqVKnzR2VM9byaslxf8l+xD7K375WIuDRmX4uJivD9kLBYtnY3klNMoLi5G5NETOPRHJGQymdjhURkwlEcfvKgYUbVqVURGRr6wnerVq2Pfvn2vFYtOFZrJkyejUaNGyMjIQGJiImxtbeHr64uUlBSd3jQkJATZ2dkah4mprU5tvK6ioiIkJ9/E2bMXMOWbOYi/cAljx43QawxiWbxoFrp29cfbnfvj7t1UscMpE+amJqjmYAMP9zfwaYeGqOdqj/DYZDhZV0BRiRI5BYUa92flF8DRpgIAwOl//1/b6e8OtYO1HBUt5UjNlu6wU2ZmFoqLi+Hi6qRx3sXFGWnp90WKSnjGmve/nT+XgA5+vVCzSnM0rOuLd/t8BAeHirh187bYoZUpo/28DWTIyZDo1KE5ceIEwsLC4OTkhDp16mD37t0ICAhAmzZtcP369Zdup7QlYWL/q8FEZgL5S+5GKGWLF81Cr15d0DlgAG6Ws7/Y/kmpUqGwRIkGbhVhZiLDqZt//8V280EuUnOeoEnlpxs/NavioD7/TPaTQjx6ooC7vRWkqqioCGfOxKNjBz/1OZlMho4d/BBTjueLGWvez5Obk4cHDx6iVu3qaNqskXricHnBz5ue0WnI6cmTJzAz+/slMpkMK1aswNixY9GuXTuEh4eXeYBCmDXzSxw4+Cdu374LWxsbvPdeb7Rr54Nu3QeLHZqgflgyG++91xt9+n6I3Nw8uP5vfDk7OxcFBdr7skjFkj8T4FvbFW52lnhcWIz9CXdw+lYmlg/0hW0Fc7zTpAYWHL4A+woWsJabYc4f8fCs7ADP/3Voqjvaon09d8w9FI9vujaDjYU5lhxNQA1HW3hVl/YY/MLFP2HtmoWIOxOP2Niz+HTcSFhbW2Ld+s1ihyYoY8jb2toKNWv9va9M9RpV0KhxAzx8+Ah376SiZ+8ueJCZhTt3UuHhUQ/ffvc19u05jKNH/hIxamEYw+f9b4Yy5GRIdOrQ1K9fH6dPn0aDBg00zi9duhQA0LNnz7KLTEDOzk74vzWL4O7uguzsXFy4eBndug9GRMTxF79YwkaPHg4AOBLxm8b5ESMmYMPPW8QIqUxkPVZgyu44ZOYVwEZuhnou9lg+0Bc+/1uCPfHtxpDJgM+3n0RhiRJv1XTBV12aarQxq0cLzD98AeO2RMNEBrSo6oTl770Fc1NpLwTcunUXnJ0cMG3qRLi5OeP8+QR06z4EGRmZL36xhBlD3k2bNcLv+35Rfz0r7CsAwK8bt2PcmC/h6uaMmbND4OziiPS0+9i8aScWfLdcrHAFZQyfN72YTvvQhIWF4fjx48+duPPJJ59g5cqVUCp1H4QTex8asRjINkB6J/Q+NIZK3/vQkLj0tQ+NodH3PjSGQp/70GT1aidY2w6/v3gSryHixnoiM5Bvv96xQ0PGgB0a48IOjbj4cEoiIiKJUUl4NZJQpD1BgIiIiAis0BAREUkPKzRa2KEhIiKSGA45aeOQExEREUkeKzRERERSwwqNFlZoiIiISPJYoSEiIpIYzqHRxgoNERERSR4rNERERBLDCo02VmiIiIhI8lihISIikhhWaLSxQ0NERCQ1KpnYERgcDjkRERGR5LFCQ0REJDEcctLGCg0RERFJHis0REREEqNScg7Nv7FCQ0RERJLHCg0REZHEcA6NNlZoiIiISPJYoSEiIpIYFfeh0cIODRERkcRwyEkbh5yIiIhI8lihISIikhgu29bGCg0RERFJHis0REREEqNSiR2B4TGYDo1MZpzlM6WR/lTajvxZ7BBEEejWTOwQRHEw/ZzYIYjiUUG+2CEQGQ2D6dAQERHRy+EcGm2cQ0NERESSxwoNERGRxLBCo40dGiIiIokx0umX/4lDTkRERCR5rNAQERFJDIectLFCQ0RERJLHCg0REZHE8Gnb2lihISIiIsljhYaIiEhiVEqxIzA8rNAQERGR5LFCQ0REJDFKzqHRwg4NERGRxHBSsDYOOREREZHksUJDREQkMdxYTxsrNERERCR57NAQERFJjEol3KGLsLAweHl5wdbWFi4uLujduzcSExM17ikoKEBQUBAcHR1hY2ODvn37Ij09XeOelJQUdOvWDVZWVnBxccGkSZNQXFysUyzs0BAREdEriYyMRFBQEGJiYnDo0CEUFRWhc+fOyM/PV98zYcIE7N69G1u3bkVkZCTu3buHPn36qK+XlJSgW7duKCwsxIkTJ7B+/XqsW7cOU6dO1SkWmUplGA8hl1eoKnYIoihRcnckYxLo1kzsEERxMP2c2CGIQmkYf72SnhQX3tXbe12q3U2wtj2S977ya+/fvw8XFxdERkaibdu2yM7OhrOzM8LDw9GvXz8AwJUrV9CgQQNER0ejdevW2L9/P7p374579+7B1dUVALBy5UpMnjwZ9+/fh4WFxUu9Nys0REREpKZQKJCTk6NxKBSKl3ptdnY2AMDBwQEAEBcXh6KiIvj7+6vvqV+/PqpVq4bo6GgAQHR0NBo3bqzuzABAQEAAcnJykJCQ8NJxs0NDREQkMUqVTLAjLCwM9vb2GkdYWNiLY1IqMX78ePj6+qJRo0YAgLS0NFhYWKBixYoa97q6uiItLU19zz87M8+uP7v2srhsm4iISGKE3FgvJCQEwcHBGufkcvkLXxcUFISLFy8iKipKqND+Ezs0REREpCaXy1+qA/NPY8eOxZ49e3Ds2DFUqVJFfd7NzQ2FhYV49OiRRpUmPT0dbm5u6ntOnTql0d6zVVDP7nkZHHIiIiKSGENZtq1SqTB27Fjs2LEDR44cQc2aNTWut2jRAubm5oiIiFCfS0xMREpKCnx8fAAAPj4+uHDhAjIyMtT3HDp0CHZ2dvDw8HjpWFihISIiolcSFBSE8PBw/P7777C1tVXPebG3t4elpSXs7e0xYsQIBAcHw8HBAXZ2dhg3bhx8fHzQunVrAEDnzp3h4eGBoUOHYu7cuUhLS8OUKVMQFBSkU6WIHRoiIiKJMZSnba9YsQIA0L59e43za9euxfvvvw8AWLhwIUxMTNC3b18oFAoEBARg+fLl6ntNTU2xZ88ejBkzBj4+PrC2tsbw4cMxY8YMnWLhPjQi4z40xoX70BgX7kNjXPS5D8256j0Fa7vprV2CtS0ko5xDM2XKBCgKbmsc8ef/FDssvRkzejiSrsYgLycZJ6J2w6tlU7FD0ovynvfqv9Zgd8oerWP0zNFwqeJS6rXdKXvg281X7NAFNWliEAoVdzB//jSxQ9GL8v5z/jzGlrdKJRPskCqjHXJKSEhEYNeB6q91fWaEVPXv3xPz54Xik6AvcSr2LD4d9xH27d0Ij0Ztcf/+A7HDE4wx5B3cYwJMTP/+N0r1N6tjVvi3iNr7FzLvZWJoiyEa93cZ1AXvfNwHcX/G6TtUvWnRogk+GjkY8fGXxA5FL4zh57w0xpo3aTLKCg3wtAOTnn5ffTx48FDskPRiwmcjsXpNONZv2ILLl6/hk6Av8fjxE3zw/ntihyYoY8g7JysHj+4/Uh9enVrh3s17uBhzAUqlUuPao/uP0DrAB1F7olDwuEDs0AVhbW2FDet/wJgxX+Dhw2yxw9ELY/g5L40x5m0oq5wMidF2aOrUqYkb10/jyuUorFu3BFWrVhI7JMGZm5ujeXNPRBw5rj6nUqkQcSQKrVu3EDEyYRlj3mbmZujwTnsc3nyo1Ou1G9dG7Ua1cWjzH/oNTI+WLP4W+/ZH4MgRcTb50jdj/DkHjDdvIXcKliqdh5wuX76MmJgY+Pj4oH79+rhy5QoWL14MhUKBIUOGoGPHji9sQ6FQaD0XQqVSQSbTzzcy9tRZfDQyGFevJsPdzRVffz0eERG/oXlzf+Tl5b+4AYlycnKAmZkZMtIzNc5nZNxH/TdrixSV8Iwx79YBrWFtZ4OIbRGlXu/8bmekXEvBlbgreo5MPwb074lmzRrD5y3hHuBnaIzx5xww3rxJm04dmgMHDqBXr16wsbHB48ePsWPHDgwbNgxNmjSBUqlE586d8ccff7ywUxMWFobp06drnDMxtYWZmb3uGbyCg38cVf/3xYtXcCr2LK5djUa/ft2xbt1mvcRAJKS33+2MuKNxyErP0rpmIbdA217tsHlJ+fxZr1LFHQsWTEfXroNe+oF6RFIj5cm7QtFpyGnGjBmYNGkSHjx4gLVr12LQoEEYOXIkDh06hIiICEyaNAlz5sx5YTshISHIzs7WOExN7V45ideVnZ2Da9duoHbtGqLFoA+ZmVkoLi6Gi6uTxnkXF2ekpd8XKSrhGVvezpWd0cSvCf749WCp1327+UJuKceR30qv3khd8+aecHV1xsmT+/E4/yYe599Eu3Y+GBv0IR7n34SJSfkcaTe2n/NnjDVv0qbTn+yEhAT1RjkDBgxAbm4u+vXrp74+ePBgxMfHv7AduVwOOzs7jUNfw02lsba2Qq1a1ZGWmvHimyWsqKgIZ87Eo2MHP/U5mUyGjh38EBNTfle6GFve/gPeRvaDbMQeiS31+tvvdsapw6eQk5Wj58j048iRKDRr1gleXgHq4/Tpc/j11x3w8gqAspzu/WRsP+fPGGvenEOjTec5NM86HiYmJqhQoQLs7f8eJrK1tUV2tuGvJpgTNgV79x1GSsoduLu7Yuo3wSgpKcHmLb+LHZrgFi7+CWvXLETcmXjExp7Fp+NGwtraEuvWl8/hh2eMJW+ZTAb//v44si0CyhLtX9zu1d3R0Lshpg+fpv/g9CQvLx8JlxI1zuXnP8GDrIda58sbY/k5/zdjzZs06dShqVGjBq5du4batZ9OtIqOjka1atXU11NSUuDu7l62EQqgcmV3bFi/FI6OFXH/fhZOnIhF23a9kJmpPd+gvNm6dRecnRwwbepEuLk54/z5BHTrPgQZGZkvfrGEGUveTf2awqWKCw49Z3WT/7tv40FqJs4eO6vnyEgfjOXn/N+MMW8Jr64WjE6PPli5ciWqVq2Kbt1KXznw1VdfISMjA6tXr9Y5ED76gIwBH31gXPjoA+Oiz0cfxFTqI1jbre9tF6xtIelUoRk9evR/Xp89e/ZrBUNEREQvJuW5LkIx2kcfEBERSRWXbWsrn+sXiYiIyKiwQkNERCQxnH2pjRUaIiIikjxWaIiIiCRGBc6h+TdWaIiIiEjyWKEhIiKSGCW3ONLCCg0RERFJHis0REREEqPkHBotrNAQERGR5LFCQ0REJDFc5aSNHRoiIiKJ4cZ62jjkRERERJLHCg0REZHEcMhJGys0REREJHms0BAREUkM59BoY4WGiIiIJI8VGiIiIolhhUYbKzREREQkeazQEBERSQxXOWljh4aIiEhilOzPaOGQExEREUkeKzREREQSw6dta2OFhoiIiCSPFRoiIiKJUYkdgAFihYaIiIgkz2AqNCVKbhNE5d/+tLNihyCKmvZuYocgihvZaWKHQOUUf2NqY4WGiIiIJM9gKjRERET0cpQyrnL6N3ZoiIiIJIaTgrVxyImIiIgkjxUaIiIiieGkYG2s0BAREZHksUJDREQkMXw4pTZWaIiIiEjyWKEhIiKSGD6cUhsrNERERCR5rNAQERFJDPeh0cYKDRERkcQoZcIdujh27Bh69OiBSpUqQSaTYefOnRrX33//fchkMo2jS5cuGvdkZWVh8ODBsLOzQ8WKFTFixAjk5eXp/D1hh4aIiIheSX5+Ppo0aYJly5Y9954uXbogNTVVffz6668a1wcPHoyEhAQcOnQIe/bswbFjxzBq1CidY+GQExERkcQYysZ6gYGBCAwM/M975HI53NzcSr12+fJlHDhwALGxsWjZsiUA4IcffkDXrl0xf/58VKpU6aVjYYWGiIiI1BQKBXJycjQOhULxyu0dPXoULi4uePPNNzFmzBg8ePBAfS06OhoVK1ZUd2YAwN/fHyYmJjh58qRO78MODRERkcSoBDzCwsJgb2+vcYSFhb1SnF26dMGGDRsQERGB7777DpGRkQgMDERJSQkAIC0tDS4uLhqvMTMzg4ODA9LS0nR6Lw45ERERkVpISAiCg4M1zsnl8ldq67333lP/d+PGjeHp6YnatWvj6NGj6NSp02vF+W/s0BAREUmMkI8+kMvlr9yBeZFatWrByckJSUlJ6NSpE9zc3JCRkaFxT3FxMbKysp477+Z5OOREREREenHnzh08ePAA7u7uAAAfHx88evQIcXFx6nuOHDkCpVIJb29vndpmhYaIiEhiDGWVU15eHpKSktRf37hxA+fOnYODgwMcHBwwffp09O3bF25ubkhOTsYXX3yBOnXqICAgAADQoEEDdOnSBSNHjsTKlStRVFSEsWPH4r333tNphRPACg0REZHkKAU8dHH69Gk0a9YMzZo1AwAEBwejWbNmmDp1KkxNTREfH4+ePXuiXr16GDFiBFq0aIHjx49rDGlt3LgR9evXR6dOndC1a1f4+flh1apVOn9PZCqVyiB2UDazqCx2CEQkkJr2uo2Flxc3snVbpUHSVlx4V2/v9WOVIYK1/fGdXwRrW0gcciIiIpIYFR+2rYVDTkRERCR5rNAQERFJjKFMCjYkrNAQERGR5LFCQ0REJDGs0Ggz6grNmNHDkXQ1Bnk5yTgRtRteLZuKHZJeMG/mXZ4Mer8fdh/dhLPXI3H2eiS27FuLtp3eAgBUruqOa/fjSj269PQXOXJhlPfP+3mMNW/6m9F2aPr374n580Ixc9b38PLugvPxl7Bv70Y4OzuKHZqgmDfzLm95p91Lx/xZP6C3/xC84z8U0VGxWLHhe9R5sxZS76bDp2FnjWPxnJXIy8vHsYi/xA69zBnD510aY8xbyIdTSlWZ7EOjUqkgk73eGjJ970NzImo3Yk+fx2fjpwAAZDIZbl6PxbLlazF33jK9xqJPzJt5i5G3vvehib16BN9NX4xtG3/Xuvb7kY1IiL+Cr8bPFDwOfe9DYyift74ZSt763IdmcTXh9qH5LEWa+9CUSYVGLpfj8uXLZdGUXpibm6N5c09EHDmuPqdSqRBxJAqtW7cQMTJhMW/mXd7zNjExQbfenWFlZYlzsfFa1xt61odH4/rYWkpHR+qM8fMGjDdv0qbTpOB/P078mZKSEsyZMweOjk/Le99///1/tqNQKKBQKDTOlUWV52U5OTnAzMwMGemZGuczMu6j/pu19RKDGJg38wbKZ971GtTBlv1rIZdb4HH+E3zy/kQkXb2hdV//wb2RlHgdZ0vp7EidMX3e/2SseXNSsDadOjSLFi1CkyZNULFiRY3zKpUKly9fhrW19Ut1SsLCwjB9+nSNczITG8hM7XQJh4gIAHAj6SZ6dhgIW1sbdOnpj7k/TMfgXiM1OjXyCnL06NsFyxasFjFSIhKKTh2a2bNnY9WqVViwYAE6duyoPm9ubo5169bBw8PjpdoJCQnRqva84Vhfl1BeS2ZmFoqLi+Hi6qRx3sXFGWnp9/UWh74xb+YNlM+8i4qKkXLjDgAgIf4KGjf1wPBRA/HNxNnqe7r06IQKlhWwc8sescIUlDF93v9krHmzQqNNpzk0X375JTZv3owxY8Zg4sSJKCoqeqU3lcvlsLOz0zj0NdwEAEVFRThzJh4dO/ipz8lkMnTs4IeYmDi9xaFvzJt5G0PewNO5NBZyC41z/Qf3wpGDkch68EicoARmrJ+3seZN2nTeWM/LywtxcXEICgpCy5YtsXHjRr12RsrKwsU/Ye2ahYg7E4/Y2LP4dNxIWFtbYt36zWKHJijmzbzLW96fTxmLYxF/4d6dNFjbWKNH3y7w9m2BDweMVd9TrWYVePk0x0cDPxUxUuEZw+ddGmPMW8rLq4XySjsF29jYYP369di0aRP8/f1RUlJS1nEJbuvWXXB2csC0qRPh5uaM8+cT0K37EGRkZL74xRLGvJl3ecvb0ekNzF06Ay6uTsjNycOVS9fw4YCx+CvypPqefoN6Ie1eBqL+jBExUuEZw+ddGmPNmzS99j40d+7cQVxcHPz9/WFtbf3K7eh7Hxoi0h9970NjKPS9Dw2JS5/70MytLtw+NF/ckuY+NK/9LKcqVaqgSpUqZRELERERvQROCtZmtI8+ICIiovKDT9smIiKSGE4K1sYKDREREUkeKzREREQSo2SNRgsrNERERCR5rNAQERFJDFc5aWOFhoiIiCSPFRoiIiKJ4QwabezQEBERSQyHnLRxyImIiIgkjxUaIiIiiVHKxI7A8LBCQ0RERJLHCg0REZHEcGM9bazQEBERkeSxQkNERCQxrM9oY4WGiIiIJI8VGiIiIonhPjTaWKEhIiIiyWOFhoiISGK4ykkbOzREREQSw+6MNg45ERERkeSxQkNERCQxnBSsjRUaIiIikjxWaIiIiCSGk4K1sUJDREREkscKDRERkcSwPqONHRoiEtyN7DSxQxBFoFszsUMQxf60s2KHQEaIHRoiIiKJ4SonbezQEBERSYyKg05aOCmYiIiIJI8VGiIiIonhkJM2VmiIiIhI8lihISIikhhurKeNFRoiIiJ6JceOHUOPHj1QqVIlyGQy7Ny5U+O6SqXC1KlT4e7uDktLS/j7++PatWsa92RlZWHw4MGws7NDxYoVMWLECOTl5ekcCzs0REREEqMS8NBFfn4+mjRpgmXLlpV6fe7cuViyZAlWrlyJkydPwtraGgEBASgoKFDfM3jwYCQkJODQoUPYs2cPjh07hlGjRukYCYeciIiI6BUFBgYiMDCw1GsqlQqLFi3ClClT0KtXLwDAhg0b4Orqip07d+K9997D5cuXceDAAcTGxqJly5YAgB9++AFdu3bF/PnzUalSpZeOhRUaIiIiiVFCJdihUCiQk5OjcSgUCp1jvHHjBtLS0uDv768+Z29vD29vb0RHRwMAoqOjUbFiRXVnBgD8/f1hYmKCkydP6vR+7NAQERFJjFLAIywsDPb29hpHWFiYzjGmpT195Imrq6vGeVdXV/W1tLQ0uLi4aFw3MzODg4OD+p6XxSEnIiIiUgsJCUFwcLDGOblcLlI0L48dGiIiIokR8tEHcrm8TDowbm5uAID09HS4u7urz6enp6Np06bqezIyMjReV1xcjKysLPXrXxaHnIiIiKjM1axZE25uboiIiFCfy8nJwcmTJ+Hj4wMA8PHxwaNHjxAXF6e+58iRI1AqlfD29tbp/VihISIikhhDefRBXl4ekpKS1F/fuHED586dg4ODA6pVq4bx48dj1qxZqFu3LmrWrIlvvvkGlSpVQu/evQEADRo0QJcuXTBy5EisXLkSRUVFGDt2LN577z2dVjgB7NAQERHRKzp9+jQ6dOig/vrZ3Jvhw4dj3bp1+OKLL5Cfn49Ro0bh0aNH8PPzw4EDB1ChQgX1azZu3IixY8eiU6dOMDExQd++fbFkyRKdY5GpVCqD2D/ZzKKy2CEQEZWpQLdmYocgiv1pZ8UOQRTFhXf19l4f1OgrWNtrb/4mWNtC4hwaIiIikjwOOREREUmMocyhMSTs0BAREUmM0jBmixgUDjkRERGR5LFCQ0REJDGsz2hjhYaIiIgkjxUaIiIiiVGyRqOFFRoiIiKSPFZoiIiIJEbIh1NKFSs0REREJHlG3aEZM3o4kq7GIC8nGSeidsOrZVOxQ9IL5s28jUF5z3v1X2uwO2WP1jF65mj1PW82r49Zv36LrVe2YXPCFoRtnQMLuYWIUQunvH/e/6YU8JAqo+3Q9O/fE/PnhWLmrO/h5d0F5+MvYd/ejXB2dhQ7NEExb+bNvMuH4B4TMLTFEPUxZdDXAICovX8BeNqZmb5hOs4dP4vPewYjuMcE7F2/B0qVlH9llc4YPu9/U0Il2CFVRvtwyhNRuxF7+jw+Gz8FACCTyXDzeiyWLV+LufOW6TUWfWLezJt56y9vfT6c8qPQkfDq5IWP244CAMzbOR/njp/DxgW/6C2GZ/T9cEpD+bz1+XDK/tV7Cdb21lu/C9a2kIyyQmNubo7mzT0RceS4+pxKpULEkSi0bt1CxMiExbyZN/Mun3mbmZuhwzvtcXjzIQCAvaM96jevj+wHjzB3+zxsiPsZYVvC4OHlIW6gAjDGzxt4OilYqP9J1Wt1aPLz87F27Vp8/fXXWLp0KR48ePBSr1MoFMjJydE49FkocnJygJmZGTLSMzXOZ2Tch5urs97i0DfmzbwB5l0etQ5oDWs7G0RsiwAAuFVzAwAMnDAIB389iGnDQpF8MRmzwr+Fe41KYoZa5ozx86bS6dSh8fDwQFZWFgDg9u3baNSoESZMmIBDhw4hNDQUHh4euHHjxgvbCQsLg729vcahUua+WgZEREbu7Xc7I+5oHLLSn/79LDORAQAObDyAiK2HcT3hOlbPWI071+/g7XffFjNUKiOcFKxNpw7NlStXUFxcDAAICQlBpUqVcOvWLZw6dQq3bt2Cp6cnvv766xe2ExISguzsbI1DZmL7ahm8gszMLBQXF8PF1UnjvIuLM9LS7+stDn1j3swbYN7ljXNlZzTxa4I/fj2oPvcw4yEA4Pa1FI177yTdhnOl8lW1MLbPm57vlYecoqOjMW3aNNjb2wMAbGxsMH36dERFRb3wtXK5HHZ2dhqHTCZ71VB0VlRUhDNn4tGxg5/6nEwmQ8cOfoiJidNbHPrGvJk38y5/efsPeBvZD7IReyRWfS79djoepD1A5VpVNO6tVLMyMu5m6DtEQRnb5/2MSqUS7JAqnXcKftbxKCgogLu7u8a1ypUr4/59afSIFy7+CWvXLETcmXjExp7Fp+NGwtraEuvWbxY7NEExb+bNvMsPmUwG//7+OLItAsoSzcGC7T/+hkETBuPG5Ru4kXAdHft1QpU6VTBnTJhI0QrHWD5v+m86d2g6deoEMzMz5OTkIDExEY0aNVJfu3XrFhwdpbHuf+vWXXB2csC0qRPh5uaM8+cT0K37EGRkZL74xRLGvJk38y4/mvo1hUsVFxz63+qmf9q1Zhcs5Bb4aOpHsK1oixuXbmDq4G+QditNhEiFZSyf9z9Jeb8Yoei0D8306dM1vm7dujUCAgLUX0+aNAl37tzBr7/+qnMg+t6HhohIaPrch8aQ6HsfGkOhz31oelTrLljbu1P2CNa2kHSq0ISGhv7n9Xnz5r1WMERERESvgk/bJiIikhgpb4AnFKPcKZiIiIjKF1ZoiIiIJIaTgrWxQkNERESSxwoNERGRxEh5AzyhsEJDREREkscKDRERkcRI+SGSQmGHhoiISGK4bFsbh5yIiIhI8lihISIikhgu29bGCg0RERFJHis0REREEsNl29pYoSEiIiLJY4WGiIhIYjiHRhsrNERERCR5rNAQERFJDPeh0cYODRERkcQoOSlYC4eciIiISPJYoSEiIpIY1me0sUJDREREkscKDRERkcRw2bY2VmiIiIhI8lihISIikhhWaLSxQkNERESSxwoNERGRxPDhlNpYoSEiIiLJY4WGiEgg+9POih2CKHyc64sdQrnHOTTa2KEhIiKSGD7LSRuHnIiIiOiVTJs2DTKZTOOoX//vCl1BQQGCgoLg6OgIGxsb9O3bF+np6YLEwg4NERGRxKhUKsEOXTVs2BCpqanqIyoqSn1twoQJ2L17N7Zu3YrIyEjcu3cPffr0KctvhRqHnIiIiEhNoVBAoVBonJPL5ZDL5aXeb2ZmBjc3N63z2dnZWLNmDcLDw9GxY0cAwNq1a9GgQQPExMSgdevWZRo3KzREREQSo4RKsCMsLAz29vYaR1hY2HNjuXbtGipVqoRatWph8ODBSElJAQDExcWhqKgI/v7+6nvr16+PatWqITo6usy/J6zQEBERkVpISAiCg4M1zj2vOuPt7Y1169bhzTffRGpqKqZPn442bdrg4sWLSEtLg4WFBSpWrKjxGldXV6SlpZV53OzQEBERSYyQG+v91/DSvwUGBqr/29PTE97e3qhevTq2bNkCS0tLoUIsFYeciIiIqExUrFgR9erVQ1JSEtzc3FBYWIhHjx5p3JOenl7qnJvXxQ4NERGRxAg5h+Z15OXlITk5Ge7u7mjRogXMzc0RERGhvp6YmIiUlBT4+Pi87rdAC4eciIiIJMZQNtabOHEievTogerVq+PevXsIDQ2FqakpBg4cCHt7e4wYMQLBwcFwcHCAnZ0dxo0bBx8fnzJf4QSwQ0NERESv6M6dOxg4cCAePHgAZ2dn+Pn5ISYmBs7OzgCAhQsXwsTEBH379oVCoUBAQACWL18uSCwylYE8stPMorLYIRARURkw1mc5Hb8b8eKbykgj17KvcDxzMT1GsLaFxDk0REREJHkcciIiIpIYQ5lDY0hYoSEiIiLJY4WGiIhIYpSGMf3VoLBCQ0RERJLHCg0REZHEcA6NNnZoiIiIJIZDTto45ERERESSxwoNERGRxHDISRsrNERERCR5rNAQERFJDOfQaGOFhoiIiCSPFRoiIiKJ4RwabazQEBERkeSxQkNERCQxKpVS7BAMjlFXaMaMHo6kqzHIy0nGiajd8GrZVOyQ9IJ5M29jwLzLb95Obk74ZkkI9lzcgcNJ+7Du8E9407Oexj0jJr6PnWe24HDSPizcNBdValYWKVphKKES7JAqo+3Q9O/fE/PnhWLmrO/h5d0F5+MvYd/ejXB2dhQ7NEExb+bNvMsvY8jbxt4Gy3cuRnFxMSYN+RJDO3yIZTNWIjc7V33PoE/eQ98P38H8Lxfh4x5j8eRxARZsnAMLubmIkZPQZCqVYaz9MrPQb+/5RNRuxJ4+j8/GTwEAyGQy3Lwei2XL12LuvGV6jUWfmDfzZt7MW2g+zvUFa/vjkI/Q2KsRxvYZ/9x7dp7Zgk0/bsWmH7cCAKxtrfH7uW0ImzAXEbv+FCy243cjBGv736o5NBas7ZSsC4K1LSSjrNCYm5ujeXNPRBw5rj6nUqkQcSQKrVu3EDEyYTFv5s28mbfU+XV+C4nxiZjx41TsOr8Naw6uRI9BXdXX3au5w9HVEaejzqjP5efm4/LZy2jYwkOMkElPdOrQnDlzBjdu3FB//fPPP8PX1xdVq1aFn58fNm3a9FLtKBQK5OTkaBz6LBQ5OTnAzMwMGemZGuczMu7DzdVZb3HoG/Nm3gDzLq+MJW/3au7oNbQn7ty4i88HfYmdG3bjsxlj0aV/ZwCAo8sbAICH9x9qvC4r8yEc/netPOAcGm06dWg++OADJCcnAwBWr16Njz/+GC1btsTXX38NLy8vjBw5Ev/3f//3wnbCwsJgb2+vcaiUuS98HRERGTcTExmuXryGVXPW4FpCEnZv3Ivd4XvRa2gPsUMjkem0bPvatWuoW7cuAGD58uVYvHgxRo4cqb7u5eWFb7/9Fh9++OF/thMSEoLg4GCNc284Cjfm+m+ZmVkoLi6Gi6uTxnkXF2ekpd/XWxz6xryZN8C8yytjyftBRhZuXb2lce5WUgradW37v+tPKzNvOL+BBxlZ6nscnN7AtYRk/QUqMAOZ/mpQdKrQWFlZITPzaTnz7t27aNWqlcZ1b29vjSGp55HL5bCzs9M4ZDKZLqG8lqKiIpw5E4+OHfzU52QyGTp28ENMTJze4tA35s28mTfzlroLsRdRtXZVjXNVa1VB2t10AEBqSioepD9AC7/m6utWNlZo0KwBEuIu6TVW0i+dKjSBgYFYsWIFVq9ejXbt2mHbtm1o0qSJ+vqWLVtQp06dMg9SCAsX/4S1axYi7kw8YmPP4tNxI2FtbYl16zeLHZqgmDfzZt7llzHkveWn37Di9yUYOm4Qjuw+igZN66PH4G6Y98XCv+9ZvR3DPx2MO9fvIPV2Gj6a9AEepGfi+MEo8QIvY3w4pTadOjTfffcdfH190a5dO7Rs2RILFizA0aNH0aBBAyQmJiImJgY7duwQKtYytXXrLjg7OWDa1Ilwc3PG+fMJ6NZ9CDIyMl/8Yglj3sybeZdfxpD3lfOJ+PqjUIz6cgSGjx+K1Nup+CF0OQ7t+HvJdPjyTbC0qoBJc4NhY2eDC7EXMHFICAoVRSJGXrb4LCdtOu9D8+jRI8yZMwe7d+/G9evXoVQq4e7uDl9fX0yYMAEtW7Z8pUD0vQ8NEREJQ8h9aAyZPvehcavYQLC20x5dFqxtIRntxnpERCQMdmiE52ov3Pc4PfuKYG0LySg31iMiIqLyhU/bJiIikhgpb4AnFFZoiIiISPJYoSEiIpIYA5n+alBYoSEiIiLJY4WGiIhIYrixnjZ2aIiIiCSGQ07aOOREREREkscKDRERkcRw2bY2VmiIiIhI8lihISIikhjOodHGCg0RERFJHis0REREEsNl29pYoSEiIiLJY4WGiIhIYlRc5aSFHRoiIiKJ4ZCTNg45ERERkeSxQkNERCQxXLatjRUaIiIikjxWaIiIiCSGk4K1sUJDREREkscKDRERkcRwDo02VmiIiIhI8tihISIikhiVSiXY8SqWLVuGGjVqoEKFCvD29sapU6fKOOMXY4eGiIhIYlQCHrravHkzgoODERoaijNnzqBJkyYICAhARkbGa2SoO5nKQAbizCwqix0CERGVAR/n+mKHIIrjdyP09l5C/s7Mz70OhUKhcU4ul0Mul5d6v7e3N7y8vLB06VIAgFKpRNWqVTFu3Dh8+eWXgsWpRWXkCgoKVKGhoaqCggKxQ9Er5s28jQHzZt6ku9DQUK3CTWhoaKn3KhQKlampqWrHjh0a54cNG6bq2bOn8MH+g8FUaMSSk5MDe3t7ZGdnw87OTuxw9IZ5M29jwLyZN+lOoVC8dIXm3r17qFy5Mk6cOAEfHx/1+S+++AKRkZE4efKk4PE+w2XbREREpPZfw0uGjJOCiYiI6JU4OTnB1NQU6enpGufT09Ph5uam11jYoSEiIqJXYmFhgRYtWiAi4u8J0UqlEhERERpDUPpg9ENOcrkcoaGhkiyvvQ7mzbyNAfNm3iS84OBgDB8+HC1btkSrVq2waNEi5Ofn44MPPtBrHEY/KZiIiIhez9KlSzFv3jykpaWhadOmWLJkCby9vfUaAzs0REREJHmcQ0NERESSxw4NERERSR47NERERCR57NAQERGR5Bl1h8YQHneub8eOHUOPHj1QqVIlyGQy7Ny5U+yQBBcWFgYvLy/Y2trCxcUFvXv3RmJiothhCW7FihXw9PSEnZ0d7Ozs4OPjg/3794sdlt7NmTMHMpkM48ePFzsUQU2bNg0ymUzjqF/fOB4SeffuXQwZMgSOjo6wtLRE48aNcfr0abHDIj0z2g6NoTzuXN/y8/PRpEkTLFu2TOxQ9CYyMhJBQUGIiYnBoUOHUFRUhM6dOyM/P1/s0ARVpUoVzJkzB3FxcTh9+jQ6duyIXr16ISEhQezQ9CY2NhY//vgjPD09xQ5FLxo2bIjU1FT1ERUVJXZIgnv48CF8fX1hbm6O/fv349KlS1iwYAHeeOMNsUMjfdProzANSKtWrVRBQUHqr0tKSlSVKlVShYWFiRiVfgHQekKqMcjIyFABUEVGRoodit698cYbqtWrV4sdhl7k5uaq6tatqzp06JCqXbt2qs8++0zskAQVGhqqatKkidhh6N3kyZNVfn5+YodBBsAoKzSFhYWIi4uDv7+/+pyJiQn8/f0RHR0tYmSkD9nZ2QAABwcHkSPRn5KSEmzatAn5+fl6345cLEFBQejWrZvGn/Py7tq1a6hUqRJq1aqFwYMHIyUlReyQBLdr1y60bNkS/fv3h4uLC5o1a4affvpJ7LBIBEbZocnMzERJSQlcXV01zru6uiItLU2kqEgflEolxo8fD19fXzRq1EjscAR34cIF2NjYQC6XY/To0dixYwc8PDzEDktwmzZtwpkzZxAWFiZ2KHrj7e2NdevW4cCBA1ixYgVu3LiBNm3aIDc3V+zQBHX9+nWsWLECdevWxcGDBzFmzBh8+umnWL9+vdihkZ4Z/bOcyLgEBQXh4sWLRjG3AADefPNNnDt3DtnZ2di2bRuGDx+OyMjIct2puX37Nj777DMcOnQIFSpUEDscvQkMDFT/t6enJ7y9vVG9enVs2bIFI0aMEDEyYSmVSrRs2RKzZ88GADRr1gwXL17EypUrMXz4cJGjI30yygqNIT3unPRn7Nix2LNnD/78809UqVJF7HD0wsLCAnXq1EGLFi0QFhaGJk2aYPHixWKHJai4uDhkZGSgefPmMDMzg5mZGSIjI7FkyRKYmZmhpKRE7BD1omLFiqhXrx6SkpLEDkVQ7u7uWh30Bg0aGMVwG2kyyg6NIT3unISnUqkwduxY7NixA0eOHEHNmjXFDkk0SqUSCoVC7DAE1alTJ1y4cAHnzp1THy1btsTgwYNx7tw5mJqaih2iXuTl5SE5ORnu7u5ihyIoX19frW0Yrl69iurVq4sUEYnFaIecDOVx5/qWl5en8S+2Gzdu4Ny5c3BwcEC1atVEjEw4QUFBCA8Px++//w5bW1v1PCl7e3tYWlqKHJ1wQkJCEBgYiGrVqiE3Nxfh4eE4evQoDh48KHZogrK1tdWaH2VtbQ1HR8dyPW9q4sSJ6NGjB6pXr4579+4hNDQUpqamGDhwoNihCWrChAl46623MHv2bAwYMACnTp3CqlWrsGrVKrFDI30Te5mVmH744QdVtWrVVBYWFqpWrVqpYmJixA5JcH/++acKgNYxfPhwsUMTTGn5AlCtXbtW7NAE9eGHH6qqV6+usrCwUDk7O6s6deqk+uOPP8QOSxTGsGz73XffVbm7u6ssLCxUlStXVr377ruqpKQkscPSi927d6saNWqkksvlqvr166tWrVoldkgkAplKpVKJ1JciIiIiKhNGOYeGiIiIyhd2aIiIiEjy2KEhIiIiyWOHhoiIiCSPHRoiIiKSPHZoiIiISPLYoSEiIiLJY4eGiIiIJI8dGiIiIpI8dmiIiIhI8tihISIiIsn7f5admP0kC2UiAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 700x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.92      0.94       415\n",
      "           1       0.99      0.83      0.90       376\n",
      "           2       0.90      0.93      0.91       330\n",
      "           3       0.78      0.90      0.83        86\n",
      "           4       0.90      1.00      0.95        37\n",
      "           5       0.76      1.00      0.86        76\n",
      "           6       0.68      1.00      0.81        60\n",
      "\n",
      "    accuracy                           0.91      1380\n",
      "   macro avg       0.85      0.94      0.89      1380\n",
      "weighted avg       0.92      0.91      0.91      1380\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "\n",
    "def print_confusion_matrix(y_true, y_pred, report=True):\n",
    "    labels = sorted(list(set(y_true)))\n",
    "    cmx_data = confusion_matrix(y_true, y_pred, labels=labels)\n",
    "    \n",
    "    df_cmx = pd.DataFrame(cmx_data, index=labels, columns=labels)\n",
    " \n",
    "    fig, ax = plt.subplots(figsize=(7, 6))\n",
    "    sns.heatmap(df_cmx, annot=True, fmt='g' ,square=False)\n",
    "    ax.set_ylim(len(set(y_true)), 0)\n",
    "    plt.show()\n",
    "    \n",
    "    if report:\n",
    "        print('Classification Report')\n",
    "        print(classification_report(y_test, y_pred))\n",
    "\n",
    "Y_pred = model.predict(X_test)\n",
    "y_pred = np.argmax(Y_pred, axis=1)\n",
    "\n",
    "print_confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FNP6aqzc9hE5"
   },
   "source": [
    "# Convert to model for Tensorflow-Lite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "ODjnYyld9hE6"
   },
   "outputs": [],
   "source": [
    "# Save as a model dedicated to inference\n",
    "model.save(model_save_path, include_optimizer=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zRfuK8Y59hE6",
    "outputId": "a4ca585c-b5d5-4244-8291-8674063209bb"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as _update_step_xla while saving (showing 1 of 1). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /tmp/tmpiu9zeepn/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /tmp/tmpiu9zeepn/assets\n",
      "2023-03-06 09:16:38.587560: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:362] Ignored output_format.\n",
      "2023-03-06 09:16:38.587581: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:365] Ignored drop_control_dependency.\n",
      "2023-03-06 09:16:38.588131: I tensorflow/cc/saved_model/reader.cc:45] Reading SavedModel from: /tmp/tmpiu9zeepn\n",
      "2023-03-06 09:16:38.589249: I tensorflow/cc/saved_model/reader.cc:89] Reading meta graph with tags { serve }\n",
      "2023-03-06 09:16:38.589263: I tensorflow/cc/saved_model/reader.cc:130] Reading SavedModel debug info (if present) from: /tmp/tmpiu9zeepn\n",
      "2023-03-06 09:16:38.593187: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:357] MLIR V1 optimization pass is not enabled\n",
      "2023-03-06 09:16:38.593938: I tensorflow/cc/saved_model/loader.cc:229] Restoring SavedModel bundle.\n",
      "2023-03-06 09:16:38.625172: I tensorflow/cc/saved_model/loader.cc:213] Running initialization op on SavedModel bundle at path: /tmp/tmpiu9zeepn\n",
      "2023-03-06 09:16:38.635704: I tensorflow/cc/saved_model/loader.cc:305] SavedModel load for tags { serve }; Status: success: OK. Took 47579 microseconds.\n",
      "2023-03-06 09:16:38.651242: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6704"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Transform model (quantization)\n",
    "\n",
    "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n",
    "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
    "tflite_quantized_model = converter.convert()\n",
    "\n",
    "open(tflite_save_path, 'wb').write(tflite_quantized_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CHBPBXdx9hE6"
   },
   "source": [
    "# Inference test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "mGAzLocO9hE7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO: Created TensorFlow Lite XNNPACK delegate for CPU.\n"
     ]
    }
   ],
   "source": [
    "interpreter = tf.lite.Interpreter(model_path=tflite_save_path)\n",
    "interpreter.allocate_tensors()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "oQuDK8YS9hE7"
   },
   "outputs": [],
   "source": [
    "# Get I / O tensor\n",
    "input_details = interpreter.get_input_details()\n",
    "output_details = interpreter.get_output_details()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "2_ixAf_l9hE7"
   },
   "outputs": [],
   "source": [
    "interpreter.set_tensor(input_details[0]['index'], np.array([X_test[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s4FoAnuc9hE7",
    "outputId": "91f18257-8d8b-4ef3-c558-e9b5f94fabbf",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 37 µs, sys: 2 µs, total: 39 µs\n",
      "Wall time: 45.3 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Inference implementation\n",
    "interpreter.invoke()\n",
    "tflite_results = interpreter.get_tensor(output_details[0]['index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vONjp19J9hE8",
    "outputId": "77205e24-fd00-42c4-f7b6-e06e527c2cba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[9.6528351e-01 3.4601703e-02 2.3508295e-05 9.1302078e-05]\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "print(np.squeeze(tflite_results))\n",
    "print(np.argmax(np.squeeze(tflite_results)))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "keypoint_classification_EN.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}