mnist-digit-classification-using-class-weights.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling Unbalanced Datasets using Class Weights in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will be working with MNIST dataset to illustrate a simple strategy to handle unbalanced datasets.  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets import the required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.utils import class_weight\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets define a method to load the mnist dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_datasets():\n",
    "    (X_train, Y_train), (X_test, Y_test) = mnist.load_data()\n",
    "    return X_train, Y_train, X_test, Y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### define a method to normalize the image data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_data(trainX, testX):\n",
    "    #reshape the data\n",
    "    X_train = trainX.reshape((trainX.shape[0], 28, 28, 1))\n",
    "    X_test = testX.reshape((testX.shape[0], 28, 28, 1))\n",
    "    # converting to floats\n",
    "    train_norm = X_train.astype('float32')\n",
    "    test_norm = X_test.astype('float32')\n",
    "    # normalize in b2in 0,1\n",
    "    X_train_norm = train_norm / 255.0\n",
    "    X_test_norm = test_norm / 255.0\n",
    "    \n",
    "    return X_train_norm, X_test_norm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets define a method to convert the integer label to one hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_Y(Y_train, Y_test):\n",
    "    Y_train_transform = to_categorical(Y_train)\n",
    "    Y_test_transform = to_categorical(Y_test)\n",
    "    \n",
    "    return Y_train_transform, Y_test_transform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Here we define a very simple CNN network for digit classification task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets define a simple CNN for mnist digit classification\n",
    "def build_model():\n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(Conv2D(32, (3, 3), activation = 'relu', input_shape = (28, 28, 1)))\n",
    "    model.add(MaxPooling2D(2, 2))\n",
    "    model.add(Dropout(0.5))\n",
    "    \n",
    "    model.add(Conv2D(64, (3, 3), activation = 'relu'))\n",
    "    model.add(MaxPooling2D(2, 2))\n",
    "    model.add(Dropout(0.5))\n",
    "    \n",
    "    model.add(Conv2D(64, (3, 3), activation = 'relu'))\n",
    "    model.add(Flatten())\n",
    "    \n",
    "    model.add(Dense(128, activation = 'relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    \n",
    "    model.add(Dense(10, activation = 'softmax'))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We use the weight computing method from Scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_class_weights(Y_train):\n",
    "    weights = class_weight.compute_class_weight('balanced', np.unique(Y_train), Y_train)\n",
    "    class_weights = {}\n",
    "    \n",
    "    for i in range(10):\n",
    "        class_weights[i] = weights[i]\n",
    "    \n",
    "    return class_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets do it now... :P"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loads the datasets\n",
    "X_train, Y_train, X_test, Y_test = load_datasets()\n",
    "\n",
    "#normalizing the x values\n",
    "X_train_norm, X_test_norm = normalize_data(X_train, X_test)\n",
    "\n",
    "#transorm the y values to one hot\n",
    "Y_train_transform, Y_test_transform = transform_Y(Y_train, Y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lets first train the model without class weight and test the model on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_without_class_weight = build_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#compile the model\n",
    "model_without_class_weight.compile(optimizer = 'rmsprop', loss = 'categorical_crossentropy', metrics = ['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 54000 samples, validate on 6000 samples\n",
      "Epoch 1/10\n",
      "54000/54000 [==============================] - 22s 414us/sample - loss: 0.4136 - acc: 0.8681 - val_loss: 0.0640 - val_acc: 0.9805\n",
      "Epoch 2/10\n",
      "54000/54000 [==============================] - 26s 479us/sample - loss: 0.1289 - acc: 0.9618 - val_loss: 0.0473 - val_acc: 0.9843\n",
      "Epoch 3/10\n",
      "54000/54000 [==============================] - 28s 519us/sample - loss: 0.0987 - acc: 0.9717 - val_loss: 0.0415 - val_acc: 0.9870\n",
      "Epoch 4/10\n",
      "54000/54000 [==============================] - 36s 658us/sample - loss: 0.0895 - acc: 0.9746 - val_loss: 0.0361 - val_acc: 0.9902\n",
      "Epoch 5/10\n",
      "54000/54000 [==============================] - 38s 696us/sample - loss: 0.0846 - acc: 0.9764 - val_loss: 0.0381 - val_acc: 0.9885\n",
      "Epoch 6/10\n",
      "54000/54000 [==============================] - 38s 701us/sample - loss: 0.0846 - acc: 0.9772 - val_loss: 0.0341 - val_acc: 0.9917\n",
      "Epoch 7/10\n",
      "54000/54000 [==============================] - 37s 679us/sample - loss: 0.0878 - acc: 0.9778 - val_loss: 0.0342 - val_acc: 0.9915\n",
      "Epoch 8/10\n",
      "54000/54000 [==============================] - 39s 722us/sample - loss: 0.0930 - acc: 0.9763 - val_loss: 0.0415 - val_acc: 0.9897\n",
      "Epoch 9/10\n",
      "54000/54000 [==============================] - 38s 699us/sample - loss: 0.0995 - acc: 0.9753 - val_loss: 0.0455 - val_acc: 0.9882\n",
      "Epoch 10/10\n",
      "54000/54000 [==============================] - 38s 710us/sample - loss: 0.1018 - acc: 0.9746 - val_loss: 0.2264 - val_acc: 0.9708\n"
     ]
    }
   ],
   "source": [
    "#train the model without class weigths\n",
    "history = model_without_class_weight.fit(X_train_norm, Y_train_transform, epochs = 10, batch_size = 56, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets test the model without class weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 128us/sample - loss: 0.2503 - acc: 0.9727\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.25026279188394546, 0.9727]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_without_class_weight.evaluate(X_test_norm, Y_test_transform)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lets train the model with class weight and test the model on test data again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_with_class_weight = build_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_with_class_weight.compile(optimizer = 'rmsprop', loss = 'categorical_crossentropy', metrics = ['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the class weights\n",
    "class_weights = get_class_weights(Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 54000 samples, validate on 6000 samples\n",
      "Epoch 1/10\n",
      "54000/54000 [==============================] - 35s 639us/sample - loss: 0.3992 - acc: 0.8723 - val_loss: 0.0738 - val_acc: 0.9773\n",
      "Epoch 2/10\n",
      "54000/54000 [==============================] - 36s 659us/sample - loss: 0.1307 - acc: 0.9605 - val_loss: 0.0433 - val_acc: 0.9857\n",
      "Epoch 3/10\n",
      "54000/54000 [==============================] - 38s 713us/sample - loss: 0.0994 - acc: 0.9704 - val_loss: 0.0371 - val_acc: 0.9898\n",
      "Epoch 4/10\n",
      "54000/54000 [==============================] - 40s 736us/sample - loss: 0.0861 - acc: 0.9752 - val_loss: 0.0354 - val_acc: 0.9908\n",
      "Epoch 5/10\n",
      "54000/54000 [==============================] - 39s 727us/sample - loss: 0.0830 - acc: 0.9775 - val_loss: 0.0340 - val_acc: 0.9910\n",
      "Epoch 6/10\n",
      "54000/54000 [==============================] - 40s 739us/sample - loss: 0.0808 - acc: 0.9773 - val_loss: 0.0368 - val_acc: 0.9905\n",
      "Epoch 7/10\n",
      "54000/54000 [==============================] - 40s 741us/sample - loss: 0.0799 - acc: 0.9782 - val_loss: 0.0331 - val_acc: 0.9908\n",
      "Epoch 8/10\n",
      "54000/54000 [==============================] - 39s 725us/sample - loss: 0.0822 - acc: 0.9775 - val_loss: 0.0368 - val_acc: 0.9920\n",
      "Epoch 9/10\n",
      "54000/54000 [==============================] - 40s 741us/sample - loss: 0.0868 - acc: 0.9771 - val_loss: 0.0398 - val_acc: 0.9913\n",
      "Epoch 10/10\n",
      "54000/54000 [==============================] - 39s 731us/sample - loss: 0.0899 - acc: 0.9765 - val_loss: 0.0419 - val_acc: 0.9895\n"
     ]
    }
   ],
   "source": [
    "#train the model without class weigths\n",
    "history = model_with_class_weight.fit(X_train_norm, Y_train_transform, epochs = 10, batch_size = 56, validation_split=0.1, class_weight=class_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets test the model with class weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 141us/sample - loss: 0.0434 - acc: 0.9866\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.04339111114250263, 0.9866]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_with_class_weight.evaluate(X_test_norm, Y_test_transform)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See the improvement..  :D "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Model Type | Test Accuracy |\n",
    "|---------------------|----------------------|\n",
    "| Model without class weight | 0.9727 |\n",
    "| Model with class weight | 0.9866 |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}