{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solutions :\n",
    "\n",
    "Here are the Parameters used in this Model to achieve an Validation Accuracy of over 95% : \n",
    "\n",
    "- Epochs : 16 Epochs\n",
    "- Batch Size : 256\n",
    "- Optimizers : Adam Optimizer \n",
    "- Data Augmentation : \n",
    "    - We Augmented the Minority Class and also some Major Classes to Make sure all classes have nearly Equal weightage\n",
    "    - Class Weightage : During the Training we ask the Optimizer to Weight all classes equally despite their imbalance using the parameter we pass called as class_weights\n",
    "- Model : We have used an Dropout after the second Convolution Layer to prevent Over-fitting.\n",
    "\n",
    "[How to Set Class weights in Keras](https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/workspace/python/source_code')\n",
    "from utils import * \n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "def augmentation(name,category,filenames,labels,i):\n",
    "    # Important Constants\n",
    "    file_path = \"Dataset/Aug/\"\n",
    "    images = []\n",
    "    (h, w) = (232,232)\n",
    "    center = (w / 2, h / 2)\n",
    "    angle90 = 90\n",
    "    angle180 = 180\n",
    "    angle270 = 270\n",
    "    scale = 1.0\n",
    "    img = load_image(name , interpolation = cv2.INTER_AREA)\n",
    "    if category == 0 :\n",
    "        images.append(cv2.flip(img,0))\n",
    "        images.append(cv2.flip(img,1))\n",
    "        images.append(cv2.warpAffine(img, cv2.getRotationMatrix2D(center, angle90, scale), (h, w)))\n",
    "        images.append(cv2.warpAffine(img, cv2.getRotationMatrix2D(center, angle180, scale), (w, h)))\n",
    "        images.append(cv2.warpAffine(img, cv2.getRotationMatrix2D(center, angle270, scale), (h, w)))\n",
    "    elif category == 1 :\n",
    "        images.append(cv2.flip(img,0))\n",
    "    elif category == 2 :\n",
    "        pass\n",
    "    elif category == 3 :\n",
    "        pass\n",
    "    elif category == 4 :\n",
    "        images.append(cv2.flip(img,0))\n",
    "    elif category == 5 :\n",
    "        images.append(cv2.flip(img,0))\n",
    "    elif category == 6 :\n",
    "        images.append(cv2.flip(img,0))\n",
    "    elif category == 7 :\n",
    "        images.append(cv2.flip(img,0))\n",
    "        images.append(cv2.flip(img,1))\n",
    "        images.append(cv2.warpAffine(img, cv2.getRotationMatrix2D(center, angle90, scale), (h, w)))\n",
    "        images.append(cv2.warpAffine(img, cv2.getRotationMatrix2D(center, angle180, scale), (w, h)))\n",
    "        images.append(cv2.warpAffine(img, cv2.getRotationMatrix2D(center, angle270, scale), (h, w)))\n",
    "    for j in range(len(images)):\n",
    "        cv2.imwrite(file_path+str(i+j)+'.jpeg',images[j])\n",
    "        filenames.append(file_path+str(i+j)+'.jpeg')\n",
    "        labels.append(category)\n",
    "    i = i + len(images)\n",
    "    return i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "................................\n",
      "45169\n"
     ]
    }
   ],
   "source": [
    "filenames,labels = load_dataset(augment_fn = augmentation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[564, 564, 564, 564, 564, 564, 564, 564]\n",
      "Counter({1: 7730, 2: 7716, 4: 5992, 3: 5119, 5: 4796, 6: 4480, 7: 2724, 0: 2100})\n"
     ]
    }
   ],
   "source": [
    "# Set the Size of the Validation set\n",
    "val_filenames , val_labels = make_test_set(filenames,labels,val=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make train test set \n",
    "test = 0.2\n",
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(filenames, labels, test_size=test, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Because the Classes are Unbalanced , we will be adding weights to the class using this function.\n",
    "import numpy as np\n",
    "from sklearn.utils import class_weight\n",
    "class_weights = class_weight.compute_class_weight('balanced',np.unique(y_train),y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "y_train = tf.one_hot(y_train,depth=8)\n",
    "y_test = tf.one_hot(y_test,depth=8)\n",
    "val_labels = tf.one_hot(val_labels,depth=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make Dataset compatible with Tensorflow Data Pipelining.\n",
    "train,test,val = make_dataset((x_train,y_train,256),(x_test,y_test,32),(val_filenames,val_labels,32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Defining the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 75, 75, 64)        19264     \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 37, 37, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 33, 33, 256)       409856    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 16, 16, 288)       663840    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 15, 15, 288)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 15, 15, 272)       705296    \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 13, 13, 256)       626944    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 6, 6, 256)         0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 6, 6, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 9216)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 3584)              33033728  \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2048)              7342080   \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 8)                 16392     \n",
      "=================================================================\n",
      "Total params: 42,817,400\n",
      "Trainable params: 42,817,400\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "import tensorflow.keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Conv2D, Flatten ,Dropout, MaxPooling2D\n",
    "from tensorflow.keras import backend as K \n",
    "\n",
    "#Reset Graphs and Create Sequential model\n",
    "\n",
    "K.clear_session()\n",
    "model = Sequential()\n",
    "\n",
    "#Convolution Layers\n",
    "\n",
    "model.add(Conv2D(64, kernel_size=10,strides=3, activation='relu', input_shape=(232,232,3)))\n",
    "model.add(MaxPooling2D(pool_size=(3, 3),strides=2))\n",
    "model.add(Conv2D(256, kernel_size=5,strides=1,activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(3, 3),strides=2))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Conv2D(288, kernel_size=3,strides=1,padding='same',activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2),strides=1))\n",
    "model.add(Conv2D(272, kernel_size=3,strides=1,padding='same',activation='relu'))\n",
    "model.add(Conv2D(256, kernel_size=3,strides=1,activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(3, 3),strides=2))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Flatten())\n",
    "\n",
    "#Linear Layers \n",
    "\n",
    "model.add(Dense(3584,activation='relu'))\n",
    "model.add(Dense(2048,activation='relu'))\n",
    "model.add(Dense(8, activation='softmax'))\n",
    "\n",
    "# Print Model Summary\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/16\n",
      "128/128 [==============================] - 33s 255ms/step - loss: 2.2549 - accuracy: 0.2262 - top2_acc: 0.4311 - val_loss: 0.0000e+00 - val_accuracy: 0.0000e+00 - val_top2_acc: 0.0000e+00\n",
      "Epoch 2/16\n",
      "128/128 [==============================] - 48s 377ms/step - loss: 1.4033 - accuracy: 0.4202 - top2_acc: 0.6689 - val_loss: 1.3685 - val_accuracy: 0.4156 - val_top2_acc: 0.6800\n",
      "Epoch 3/16\n",
      "128/128 [==============================] - 68s 534ms/step - loss: 1.1005 - accuracy: 0.5502 - top2_acc: 0.7826 - val_loss: 0.8892 - val_accuracy: 0.6480 - val_top2_acc: 0.8457\n",
      "Epoch 4/16\n",
      "128/128 [==============================] - 67s 525ms/step - loss: 0.8395 - accuracy: 0.6678 - top2_acc: 0.8688 - val_loss: 0.7111 - val_accuracy: 0.7143 - val_top2_acc: 0.8936\n",
      "Epoch 5/16\n",
      "128/128 [==============================] - 69s 542ms/step - loss: 0.6006 - accuracy: 0.7714 - top2_acc: 0.9208 - val_loss: 0.5447 - val_accuracy: 0.8010 - val_top2_acc: 0.9342\n",
      "Epoch 6/16\n",
      "128/128 [==============================] - 80s 622ms/step - loss: 0.4687 - accuracy: 0.8255 - top2_acc: 0.9487 - val_loss: 0.3657 - val_accuracy: 0.8686 - val_top2_acc: 0.9676\n",
      "Epoch 7/16\n",
      "128/128 [==============================] - 75s 583ms/step - loss: 0.3544 - accuracy: 0.8702 - top2_acc: 0.9672 - val_loss: 0.3090 - val_accuracy: 0.8956 - val_top2_acc: 0.9785\n",
      "Epoch 8/16\n",
      "128/128 [==============================] - 70s 543ms/step - loss: 0.2897 - accuracy: 0.8944 - top2_acc: 0.9765 - val_loss: 0.2515 - val_accuracy: 0.9116 - val_top2_acc: 0.9807\n",
      "Epoch 9/16\n",
      "128/128 [==============================] - 82s 641ms/step - loss: 0.2332 - accuracy: 0.9154 - top2_acc: 0.9844 - val_loss: 0.2114 - val_accuracy: 0.9295 - val_top2_acc: 0.9863\n",
      "Epoch 10/16\n",
      "128/128 [==============================] - 74s 582ms/step - loss: 0.2077 - accuracy: 0.9258 - top2_acc: 0.9869 - val_loss: 0.2039 - val_accuracy: 0.9337 - val_top2_acc: 0.9878\n",
      "Epoch 11/16\n",
      "128/128 [==============================] - 69s 536ms/step - loss: 0.1667 - accuracy: 0.9397 - top2_acc: 0.9913 - val_loss: 0.1969 - val_accuracy: 0.9300 - val_top2_acc: 0.9900\n",
      "Epoch 12/16\n",
      "128/128 [==============================] - 80s 627ms/step - loss: 0.1479 - accuracy: 0.9471 - top2_acc: 0.9931 - val_loss: 0.1544 - val_accuracy: 0.9497 - val_top2_acc: 0.9922\n",
      "Epoch 13/16\n",
      "128/128 [==============================] - 74s 575ms/step - loss: 0.1381 - accuracy: 0.9512 - top2_acc: 0.9939 - val_loss: 0.1367 - val_accuracy: 0.9539 - val_top2_acc: 0.9945\n",
      "Epoch 14/16\n",
      "128/128 [==============================] - 73s 574ms/step - loss: 0.1122 - accuracy: 0.9618 - top2_acc: 0.9966 - val_loss: 0.1536 - val_accuracy: 0.9528 - val_top2_acc: 0.9938\n",
      "Epoch 15/16\n",
      "128/128 [==============================] - 73s 569ms/step - loss: 0.1091 - accuracy: 0.9622 - top2_acc: 0.9957 - val_loss: 0.1487 - val_accuracy: 0.9539 - val_top2_acc: 0.9938\n",
      "Epoch 16/16\n",
      "128/128 [==============================] - 80s 622ms/step - loss: 0.1045 - accuracy: 0.9626 - top2_acc: 0.9967 - val_loss: 0.1408 - val_accuracy: 0.9530 - val_top2_acc: 0.9949\n",
      "Test loss: 0.1385389032600192\n",
      "Test accuracy: 0.953394\n"
     ]
    }
   ],
   "source": [
    "#Define Batch Size and Number of Epochs\n",
    "epochs = 16\n",
    "import functools\n",
    "import tensorflow\n",
    "\n",
    "# Include Top-2 Accuracy Metrics \n",
    "top2_acc = functools.partial(tensorflow.keras.metrics.top_k_categorical_accuracy, k=2)\n",
    "top2_acc.__name__ = 'top2_acc'\n",
    "\n",
    "#Compile Model with Loss Function , Optimizer and Metrics\n",
    "model.compile(loss=tensorflow.keras.losses.categorical_crossentropy, \n",
    "              optimizer=tensorflow.keras.optimizers.Adam(),\n",
    "              metrics=['accuracy',top2_acc])\n",
    "\n",
    "# Train the Model \n",
    "trained_model = model.fit(train,validation_data=val,epochs=epochs,class_weight=class_weights)\n",
    "\n",
    "# Test Model Aganist Validation Set\n",
    "score = model.evaluate(test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n",
    "\n",
    "#Save Our Model and Weights\n",
    "model.save('cyc_pred_first.h5')\n",
    "model.save_weights('model_weights_first.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 38208 characters]\n",
      "text/plain": [
       "<Figure size 1080x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "f = plt.figure(figsize=(15,5))\n",
    "ax = f.add_subplot(121)\n",
    "ax.plot(trained_model.history['accuracy'])\n",
    "ax.plot(trained_model.history['val_accuracy'])\n",
    "ax.set_title('Model Accuracy')\n",
    "ax.set_ylabel('Accuracy')\n",
    "ax.set_xlabel('Epoch')\n",
    "ax.legend(['Train', 'Val'])\n",
    "\n",
    "ax2 = f.add_subplot(122)\n",
    "ax2.plot(trained_model.history['loss'])\n",
    "ax2.plot(trained_model.history['val_loss'])\n",
    "ax2.set_title('Model Loss')\n",
    "ax2.set_ylabel('Loss')\n",
    "ax2.set_xlabel('Epoch')\n",
    "ax2.legend(['Train', 'Val'],loc= 'upper left')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f6764ae9978>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 32716 characters]\n",
      "text/plain": [
       "<Figure size 1080x1080 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plotting a heatmap using the confusion matrix\n",
    "pred = model.predict(val)\n",
    "p = np.argmax(pred, axis=1)\n",
    "y_valid = np.argmax(val_labels, axis=1, out=None)\n",
    "import seaborn as sn\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import pandas as pd\n",
    "results = confusion_matrix(y_valid, p) \n",
    "classes=['NC','TD','TC','H1','H3','H3','H4','H5']\n",
    "df_cm = pd.DataFrame(results, index = [i for i in classes], columns = [i for i in classes])\n",
    "plt.figure(figsize = (15,15))\n",
    "\n",
    "sn.heatmap(df_cm, annot=True, cmap=\"Blues\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
