notebooks/Tensorflow_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-level TF Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "from common.params import *\n",
    "from common.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Force one-gpu\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Performance Improvement\n",
    "# 1. Auto-tune\n",
    "# This is no longer needed with recent versions of TF\n",
    "# And actually seems to make the performance worse\n",
    "#os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Numpy:  1.14.1\n",
      "Tensorflow:  1.4.0\n",
      "GPU:  ['Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB']\n",
      "CUDA Version 8.0.61\n",
      "CuDNN Version  6.0.21\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"Tensorflow: \", tf.__version__)\n",
    "print(\"GPU: \", get_gpu_name())\n",
    "print(get_cuda_version())\n",
    "print(\"CuDNN Version \", get_cudnn_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_symbol(training, n_classes=N_CLASSES):\n",
    "    # Tensorflow requires a flag for training in dropout\n",
    "    conv1 = tf.layers.conv2d(X, activation=tf.nn.relu, filters=50, kernel_size=(3, 3), \n",
    "                             padding='same', data_format='channels_first')\n",
    "    conv2 = tf.layers.conv2d(conv1, filters=50, kernel_size=(3, 3), \n",
    "                             padding='same', data_format='channels_first')\n",
    "    pool1 = tf.layers.max_pooling2d(conv2, pool_size=(2, 2), strides=(2, 2), \n",
    "                                    padding='valid', data_format='channels_first')\n",
    "    relu2 = tf.nn.relu(pool1)\n",
    "    drop1 = tf.layers.dropout(relu2, 0.25, training=training)\n",
    "    \n",
    "    conv3 = tf.layers.conv2d(drop1, activation=tf.nn.relu, filters=100, kernel_size=(3, 3), \n",
    "                             padding='same', data_format='channels_first')\n",
    "    conv4 = tf.layers.conv2d(conv3, filters=100, kernel_size=(3, 3), \n",
    "                             padding='same', data_format='channels_first')\n",
    "    pool2 = tf.layers.max_pooling2d(conv4, pool_size=(2, 2), strides=(2, 2), \n",
    "                                    padding='valid', data_format='channels_first')\n",
    "    relu4 = tf.nn.relu(pool2)\n",
    "    drop2 = tf.layers.dropout(relu4, 0.25, training=training)   \n",
    "    \n",
    "    flatten = tf.reshape(drop2, shape=[-1, 100*8*8])\n",
    "    fc1 = tf.layers.dense(flatten, 512, activation=tf.nn.relu)\n",
    "    drop3 = tf.layers.dropout(fc1, 0.5, training=training)\n",
    "    logits = tf.layers.dense(drop3, n_classes, name='output')\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model(m, y, learning_rate=LR, momentum=MOMENTUM):\n",
    "    # Single-class labels, don't need dense one-hot\n",
    "    # Expects unscaled logits, not output of tf.nn.softmax\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=m, labels=y)\n",
    "    loss = tf.reduce_mean(xentropy)\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "    return training_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing train set...\n",
      "Preparing test set...\n",
      "(50000, 3, 32, 32) (10000, 3, 32, 32) (50000,) (10000,)\n",
      "float32 float32 int32 int32\n",
      "CPU times: user 670 ms, sys: 556 ms, total: 1.23 s\n",
      "Wall time: 1.23 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Data into format for library\n",
    "x_train, x_test, y_train, y_test = cifar_for_library(channel_first=True)\n",
    "print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
    "print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 115 ms, sys: 3.38 ms, total: 118 ms\n",
      "Wall time: 117 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Place-holders\n",
    "X = tf.placeholder(tf.float32, shape=[None, 3, 32, 32])\n",
    "y = tf.placeholder(tf.int32, shape=[None])\n",
    "training = tf.placeholder(tf.bool)  # Indicator for dropout layer\n",
    "# Initialise model\n",
    "sym = create_symbol(training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 463 ms, sys: 365 ms, total: 828 ms\n",
      "Wall time: 823 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model = init_model(sym, y)\n",
    "init = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "# Accuracy logging\n",
    "correct = tf.nn.in_top_k(sym, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.484375\n",
      "1 Train accuracy: 0.53125\n",
      "2 Train accuracy: 0.640625\n",
      "3 Train accuracy: 0.75\n",
      "4 Train accuracy: 0.703125\n",
      "5 Train accuracy: 0.6875\n",
      "6 Train accuracy: 0.75\n",
      "7 Train accuracy: 0.84375\n",
      "8 Train accuracy: 0.78125\n",
      "9 Train accuracy: 0.796875\n",
      "CPU times: user 42.2 s, sys: 10.5 s, total: 52.7 s\n",
      "Wall time: 56.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Main training loop: 57s\n",
    "for j in range(EPOCHS):\n",
    "    for data, label in yield_mb(x_train, y_train, BATCHSIZE, shuffle=True):\n",
    "        sess.run(model, feed_dict={X: data, y: label, training: True})\n",
    "    # Log\n",
    "    acc_train = sess.run(accuracy, feed_dict={X: data, y: label, training: True})\n",
    "    print(j, \"Train accuracy:\", acc_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.85 s, sys: 211 ms, total: 3.06 s\n",
      "Wall time: 3.07 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Main evaluation loop: 3.22s\n",
    "n_samples = (y_test.shape[0]//BATCHSIZE)*BATCHSIZE\n",
    "y_guess = np.zeros(n_samples, dtype=np.int)\n",
    "y_truth = y_test[:n_samples]\n",
    "c = 0\n",
    "for data, label in yield_mb(x_test, y_test, BATCHSIZE):\n",
    "    pred = tf.argmax(sym,1)\n",
    "    output = sess.run(pred, feed_dict={X: data, training: False})\n",
    "    y_guess[c*BATCHSIZE:(c+1)*BATCHSIZE] = output\n",
    "    c += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.7716346153846154\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", 1.*sum(y_guess == y_truth)/len(y_guess))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}