notebooks/CNTK_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-level RNN CNTK Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import cntk\n",
    "from cntk.layers import Embedding, LSTM, GRU, Dense, Recurrence\n",
    "from cntk import sequence\n",
    "from common.params_lstm import *\n",
    "from common.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Force one-gpu\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Numpy:  1.14.1\n",
      "CNTK:  2.4\n",
      "GPU:  ['Tesla P100-PCIE-16GB', 'Tesla P100-PCIE-16GB']\n",
      "CUDA Version 8.0.61\n",
      "CuDNN Version  6.0.21\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"CNTK: \", cntk.__version__)\n",
    "print(\"GPU: \", get_gpu_name())\n",
    "print(get_cuda_version())\n",
    "print(\"CuDNN Version \", get_cudnn_version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_symbol(CUDNN=True, edim=EMBEDSIZE, nhid=NUMHIDDEN):\n",
    "    # Weight initialiser from uniform distribution\n",
    "    # Activation (unless states) is None\n",
    "    with cntk.layers.default_options(init = cntk.glorot_uniform()):\n",
    "        x = Embedding(edim)(features) # output: list of len=BATCHSIZE of arrays with shape=(MAXLEN, EMBEDSIZE)\n",
    "        \n",
    "        # Since we have a vanilla RNN, instead of using the more flexible Recurrence(GRU) unit, which allows for\n",
    "        # example LayerNormalisation to be added to the network, we can use optimized_rnnstack which quickly\n",
    "        # goes down to the CuDNN level. This is another reason not to read much into the speed comparison because\n",
    "        # it becomes a measure of which framework has the fastest way to go down to CuDNN.\n",
    "        if not CUDNN:\n",
    "            x = Recurrence(GRU(nhid))(x) # output: list of len=BATCHSIZE of arrays with shape=(MAXLEN, NUMHIDDEN)\n",
    "        else:\n",
    "            W = cntk.parameter((cntk.InferredDimension, 4))\n",
    "            x = cntk.ops.optimized_rnnstack(x, W, nhid, \n",
    "                                            num_layers=1, bidirectional=False, recurrent_op='gru')\n",
    "        \n",
    "        x = sequence.last(x) #o utput: array with shape=(BATCHSIZE, NUMHIDDEN)\n",
    "        x = Dense(2)(x) # output: array with shape=(BATCHSIZE, 2)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model(m, labels, lr=LR, b1=BETA_1, b2=BETA_2, eps=EPS):\n",
    "    # Loss (dense labels); check if support for sparse labels\n",
    "    loss = cntk.cross_entropy_with_softmax(m, labels)  \n",
    "    # ADAM, set unit_gain to False to match others\n",
    "    learner = cntk.adam(m.parameters,\n",
    "                        lr=cntk.learning_rate_schedule(lr, cntk.UnitType.minibatch) ,\n",
    "                        momentum=cntk.momentum_schedule(b1), \n",
    "                        variance_momentum=cntk.momentum_schedule(b2),\n",
    "                        epsilon=eps,\n",
    "                        unit_gain=False)\n",
    "    trainer = cntk.Trainer(m, (loss, cntk.classification_error(m, labels)), [learner])\n",
    "    return trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing train set...\n",
      "Preparing test set...\n",
      "Trimming to 30000 max-features\n",
      "Padding to length 150\n",
      "(25000, 150) (25000, 150) (25000, 2) (25000, 2)\n",
      "int32 int32 float32 float32\n",
      "CPU times: user 5.77 s, sys: 379 ms, total: 6.15 s\n",
      "Wall time: 6.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Data into format for library\n",
    "x_train, x_test, y_train, y_test = imdb_for_library(\n",
    "    seq_len=MAXLEN, max_features=MAXFEATURES, one_hot=True) # CNTK format\n",
    "y_train = y_train.astype(np.float32)\n",
    "y_test = y_test.astype(np.float32)\n",
    "print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
    "print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.6 ms, sys: 32.4 ms, total: 43 ms\n",
      "Wall time: 52.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Placeholders\n",
    "features = sequence.input_variable(shape=MAXFEATURES, is_sparse=True)\n",
    "labels = cntk.input_variable(2)\n",
    "# Load symbol\n",
    "sym = create_symbol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 110 ms, sys: 262 ms, total: 371 ms\n",
      "Wall time: 377 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "trainer = init_model(sym, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1  |  Accuracy: 0.890625\n",
      "Epoch 2  |  Accuracy: 0.875000\n",
      "Epoch 3  |  Accuracy: 0.968750\n",
      "CPU times: user 13.7 s, sys: 1.31 s, total: 15 s\n",
      "Wall time: 14.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Main training loop: 14.6s\n",
    "for j in range(EPOCHS):\n",
    "    for data, label in yield_mb(x_train, y_train, BATCHSIZE, shuffle=True):\n",
    "        data_1hot = cntk.Value.one_hot(data, MAXFEATURES) #TODO: do this externally and generate batches of 1hot\n",
    "        trainer.train_minibatch({features: data_1hot, labels: label})\n",
    "    # Log (this is just last batch in epoch, not average of batches)\n",
    "    eval_error = trainer.previous_minibatch_evaluation_average\n",
    "    print(\"Epoch %d  |  Accuracy: %.6f\" % (j+1, (1-eval_error)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.38 s, sys: 156 ms, total: 2.54 s\n",
      "Wall time: 2.54 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Main evaluation loop: 2.55s\n",
    "z = cntk.softmax(sym)\n",
    "n_samples = (y_test.shape[0]//BATCHSIZE)*BATCHSIZE\n",
    "y_guess = np.zeros(n_samples, dtype=np.int)\n",
    "y_truth = np.argmax(y_test[:n_samples], axis=-1)\n",
    "c = 0\n",
    "for data, label in yield_mb(x_test, y_test, BATCHSIZE):\n",
    "    data = cntk.Value.one_hot(data, MAXFEATURES)\n",
    "    predicted_label_probs = z.eval({features : data})\n",
    "    y_guess[c*BATCHSIZE:(c+1)*BATCHSIZE] = np.argmax(predicted_label_probs, axis=-1)\n",
    "    c += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8565304487179487\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", 1.*sum(y_guess == y_truth)/len(y_guess))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}