notebooks/Keras_Theano_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-level Keras (Theano) Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lots of warnings!\n",
    "# Not sure why Keras creates model with float64?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting /home/iliauk/.theanorc\n"
     ]
    }
   ],
   "source": [
    "%%writefile ~/.theanorc\n",
    "[global]\n",
    "device = cuda0\n",
    "force_device= True\n",
    "floatX = float32\n",
    "warn_float64 = warn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cuDNN version 6021 on context None\n",
      "Mapped name None to device cuda0: Tesla P100-PCIE-16GB (BC4B:00:00.0)\n",
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "os.environ['KERAS_BACKEND'] = \"theano\"\n",
    "import theano\n",
    "import keras as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from common.params import *\n",
    "from common.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Force one-gpu\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Performance Improvement\n",
    "# 1. Make sure channels-first (not last)\n",
    "K.backend.set_image_data_format('channels_first')\n",
    "# 2. CuDNN auto-tune\n",
    "theano.config.dnn.conv.algo_fwd = \"time_once\"\n",
    "theano.config.dnn.conv.algo_bwd_filter = \"time_once\"\n",
    "theano.config.dnn.conv.algo_bwd_data = \"time_once\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Keras:  2.1.4\n",
      "Numpy:  1.14.1\n",
      "Theano:  1.0.1\n",
      "theano\n",
      "channels_first\n",
      "GPU:  ['Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB']\n",
      "CUDA Version 8.0.61\n",
      "CuDNN Version  6.0.21\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Keras: \", K.__version__)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"Theano: \", theano.__version__)\n",
    "print(K.backend.backend())\n",
    "print(K.backend.image_data_format())\n",
    "print(\"GPU: \", get_gpu_name())\n",
    "print(get_cuda_version())\n",
    "print(\"CuDNN Version \", get_cudnn_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_symbol(n_classes=N_CLASSES):\n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(Conv2D(50, kernel_size=(3, 3), padding='same', activation='relu',\n",
    "                     input_shape=(3, 32, 32)))\n",
    "    model.add(Conv2D(50, kernel_size=(3, 3), padding='same', activation='relu'))    \n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "    \n",
    "    model.add(Conv2D(100, kernel_size=(3, 3), padding='same', activation='relu'))\n",
    "    model.add(Conv2D(100, kernel_size=(3, 3), padding='same', activation='relu'))    \n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "        \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(512, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(n_classes, activation='softmax'))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model(m, lr=LR, momentum=MOMENTUM):\n",
    "    m.compile(\n",
    "        loss = \"categorical_crossentropy\",\n",
    "        optimizer = K.optimizers.SGD(lr, momentum),\n",
    "        metrics = ['accuracy'])\n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing train set...\n",
      "Preparing test set...\n",
      "(50000, 3, 32, 32) (10000, 3, 32, 32) (50000, 10) (10000, 10)\n",
      "float32 float32 int32 int32\n",
      "CPU times: user 650 ms, sys: 604 ms, total: 1.25 s\n",
      "Wall time: 1.25 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Data into format for library\n",
    "x_train, x_test, y_train, y_test = cifar_for_library(channel_first=True, one_hot=True)\n",
    "print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
    "print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/theano/compile/function_module.py:1519: UserWarning: You are creating a TensorVariable with float64 dtype. You requested an action via the Theano flag warn_float64={ignore,warn,raise,pdb}.\n",
      "  optimizer_profile = optimizer(fgraph)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/theano/compile/function_module.py:1519: UserWarning: You are creating a TensorVariable with float64 dtype. You requested an action via the Theano flag warn_float64={ignore,warn,raise,pdb}.\n",
      "  optimizer_profile = optimizer(fgraph)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/theano/compile/function_module.py:1519: UserWarning: You are creating a TensorVariable with float64 dtype. You requested an action via the Theano flag warn_float64={ignore,warn,raise,pdb}.\n",
      "  optimizer_profile = optimizer(fgraph)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.16 s, sys: 163 ms, total: 1.33 s\n",
      "Wall time: 3.29 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/theano/compile/function_module.py:1519: UserWarning: You are creating a TensorVariable with float64 dtype. You requested an action via the Theano flag warn_float64={ignore,warn,raise,pdb}.\n",
      "  optimizer_profile = optimizer(fgraph)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/theano/compile/function_module.py:1519: UserWarning: You are creating a TensorVariable with float64 dtype. You requested an action via the Theano flag warn_float64={ignore,warn,raise,pdb}.\n",
      "  optimizer_profile = optimizer(fgraph)\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load symbol\n",
    "sym = create_symbol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.2 ms, sys: 0 ns, total: 12.2 ms\n",
      "Wall time: 12 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Initialise model\n",
    "model = init_model(sym)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 50, 32, 32)        1400      \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 50, 32, 32)        22550     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 50, 16, 16)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 50, 16, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 100, 16, 16)       45100     \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 100, 16, 16)       90100     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 100, 8, 8)         0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 100, 8, 8)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 6400)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               3277312   \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 3,441,592\n",
      "Trainable params: 3,441,592\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 8s 163us/step - loss: 1.8260 - acc: 0.3317\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 8s 158us/step - loss: 1.3923 - acc: 0.4949\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 9s 181us/step - loss: 1.1576 - acc: 0.5866\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 8s 155us/step - loss: 1.0030 - acc: 0.6450\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 8s 153us/step - loss: 0.8973 - acc: 0.6837\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 8s 155us/step - loss: 0.8053 - acc: 0.7170\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 9s 181us/step - loss: 0.7338 - acc: 0.7404\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 8s 154us/step - loss: 0.6722 - acc: 0.7627\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 8s 160us/step - loss: 0.6091 - acc: 0.7848\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 8s 160us/step - loss: 0.5626 - acc: 0.8016\n",
      "CPU times: user 42.6 s, sys: 23.5 s, total: 1min 6s\n",
      "Wall time: 1min 33s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fba1206e278>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Main training loop: 1m33s\n",
    "model.fit(x_train,\n",
    "          y_train,\n",
    "          batch_size=BATCHSIZE,\n",
    "          epochs=EPOCHS,\n",
    "          verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 886 ms, sys: 191 ms, total: 1.08 s\n",
      "Wall time: 2.47 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Main evaluation loop: 2.47s\n",
    "y_guess = model.predict(x_test, batch_size=BATCHSIZE)\n",
    "y_guess = np.argmax(y_guess, axis=-1)\n",
    "y_truth = np.argmax(y_test, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.7525\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", 1.*sum(y_guess == y_truth)/len(y_guess))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}