components/customized_op/customized_op.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Customized layer\n",
    "- combination from the tf api\n",
    "- implementation by python code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### combination from the tf api\n",
    "\n",
    "for example:\n",
    "\n",
    "#### prelu\n",
    "```\n",
    "y = prelu(x) \n",
    "if x > 0: \n",
    "    y = x\n",
    "else:\n",
    "    y = alpha*x #alpha >0\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model define\n",
    "def prelu(x, name = 'prelu'):\n",
    "    with tf.variable_scope(name):\n",
    "        alphas = tf.get_variable('alpha', x.get_shape()[-1], initializer=tf.constant_initializer(0.25), regularizer = l2_regularizer, dtype = tf.float32)\n",
    "    pos = tf.nn.relu(x)\n",
    "    neg = tf.multiply(alphas,(x - abs(x)) * 0.5)\n",
    "    return pos + neg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### implementation by python code\n",
    "- op forward implementation\n",
    "- op backward implementation (if needed)\n",
    "- ```tf.py_func``` gradient override"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implement Large Margin Cosine Loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normalize the weight and feature\n",
    "![title](images/norm_f.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normalized version of Softmax Loss (NSL)\n",
    "![title](images/norm_softmax_loss_f.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Large Margin Cosine Loss (LMCL)\n",
    "![title](images/large_margin_loss_new2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def py_func(func, inp, Tout, stateful = True, name=None, grad_func=None):\n",
    "    # Need to generate a unique name to avoid duplicates\n",
    "    rand_name = 'PyFuncGrad' + str(np.random.randint(0,1E+8))\n",
    "    tf.RegisterGradient(rand_name)(grad_func)\n",
    "    g = tf.get_default_graph()\n",
    "    with g.gradient_override_map({'PyFunc':rand_name}):\n",
    "        return tf.py_func(func,inp,Tout,stateful=stateful, name=name)\n",
    "\n",
    "\n",
    "def coco_forward(xw, y, m, name=None):\n",
    "    num = len(y)\n",
    "    orig_ind = range(num)\n",
    "    xw[orig_ind,y] -= m\n",
    "    return xw\n",
    "\n",
    "def coco_help(grad,y):\n",
    "    grad_copy = grad.copy()\n",
    "    return grad_copy\n",
    "\n",
    "def coco_backward(op, grad):\n",
    "    \n",
    "    y = op.inputs[1]\n",
    "    m = op.inputs[2]\n",
    "    grad_copy = tf.py_func(coco_help,[grad,y],tf.float32)\n",
    "    return grad_copy,y,m\n",
    "\n",
    "def coco_func(xw,y,m, name=None):\n",
    "    with tf.op_scope([xw,y,m],name,\"Coco_func\") as name:\n",
    "        coco_out = py_func(coco_forward,[xw,y,m],tf.float32,name=name,grad_func=coco_backward)\n",
    "        return coco_out\n",
    "    \n",
    "# function interface\n",
    "def cosine_loss(x,y,num_cls,reuse=False,alpha=0.25,scale=64,name='cosine_loss'):\n",
    "    '''\n",
    "    x: B x D - features\n",
    "    y: B x 1 - labels\n",
    "    num_cls: 1 - total class number\n",
    "    alpah: 1 - margin\n",
    "    scale: 1 - scaling paramter\n",
    "    '''\n",
    "    # define the classifier weights\n",
    "    xs = x.get_shape()\n",
    "    with tf.variable_scope('centers_var',reuse=reuse) as center_scope:\n",
    "        w = tf.get_variable(\"centers\", [xs[1], num_cls], dtype=tf.float32, \n",
    "            initializer=tf.contrib.layers.xavier_initializer(),trainable=True)\n",
    "   \n",
    "    #normalize the feature and weight\n",
    "    #(N,D)\n",
    "    x_feat_norm = tf.nn.l2_normalize(x,1,1e-10)\n",
    "    #(D,C)\n",
    "    w_feat_norm = tf.nn.l2_normalize(w,0,1e-10)\n",
    "    \n",
    "    # get the scores after normalization \n",
    "    #(N,C)\n",
    "    xw_norm = tf.matmul(x_feat_norm, w_feat_norm)  \n",
    "    #value = tf.identity(xw)\n",
    "    #substract the marigin and scale it\n",
    "    value = coco_func(xw_norm,y,alpha) * scale\n",
    "    \n",
    "    # compute the loss as softmax loss\n",
    "    cos_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=value))\n",
    "\n",
    "    return cos_loss \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[draw figure online](http://www.atool.org/canvas.php)\n",
    "\n",
    "[awwapp](https://awwapp.com)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## how to implement the large margin cosine loss by combining the tf.api?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}