Cross Entropy Method.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross-Entropy Method\n",
    "---\n",
    "In this notebookm we will train a Cross-Entropy Method with OpenAI Gym's MountainCarContinuous environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym \n",
    "import math\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline \n",
    "\n",
    "\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Instantiate the Environment and Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "observation space: Box(2,)\n",
      "action space: Box(1,)\n",
      " - low: [-1.]\n",
      " - high: [1.]\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "env= gym.make('MountainCarContinuous-v0')\n",
    "env.seed(101)\n",
    "np.random.seed(101)\n",
    "\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "print(' - low:', env.action_space.low)\n",
    "print(' - high:', env.action_space.high)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(nn.Module):\n",
    "    def __init__(self,env,h_size=16):\n",
    "        super().__init__()\n",
    "        self.env = env\n",
    "        # state, hidden layer, action_sizes\n",
    "        self.s_size = env.observation_space.shape[0]\n",
    "        self.h_size = h_size\n",
    "        self.a_size = env.action_space.shape[0]\n",
    "        ## define layer\n",
    "        self.fc1 = nn.Linear(self.s_size,self.h_size)\n",
    "        self.fc2 = nn.Linear(self.h_size,self.a_size)\n",
    "        \n",
    "    def set_weights(self,weights):\n",
    "        s_size = self.s_size\n",
    "        h_size = self.h_size\n",
    "        a_size =self.a_size\n",
    "        \n",
    "        # separate the weights for each layer \n",
    "        fc1_end = (s_size*h_size) + h_size\n",
    "        ## s_size*h_size is the number of weights between input and hidden layer\n",
    "        ## h_size is the number of bias weights between input and hidden layer\n",
    "        ## s_size*h_size + h_size tells us the total number of weights between input_layer and hidden_layer \n",
    "        fc1_W = torch.from_numpy(weights[:s_size*h_size].reshape(s_size,h_size))\n",
    "        fc1_b = torch.from_numpy(weights[s_size*h_size:fc1_end])\n",
    "        fc2_W = torch.from_numpy(weights[fc1_end:fc1_end+(h_size*a_size)].reshape(h_size, a_size))\n",
    "        fc2_b = torch.from_numpy(weights[fc1_end+(h_size*a_size):])\n",
    "        \n",
    "        ## Set the weights for each layer\n",
    "        self.fc1.weight.data.copy_(fc1_W.view_as(self.fc1.weight.data))\n",
    "        self.fc1.bias.data.copy_(fc1_b.view_as(self.fc1.bias.data))\n",
    "        self.fc2.weight.data.copy_(fc2_W.view_as(self.fc2.weight.data))\n",
    "        self.fc2.bias.data.copy_(fc2_b.view_as(self.fc2.bias.data))\n",
    "        \n",
    "        ## the difference between view and reshape is that in reshape\n",
    "        ## it changes the shape of original data and view return the the \n",
    "        ## data in which is requested but original data shape is same\n",
    "    def get_weights_dim(self):\n",
    "        ## plus 1 compensate the shape for bias\n",
    "        return (self.s_size+1)*self.h_size + (self.h_size+1)*self.a_size\n",
    "    \n",
    "    def forward(self,x):\n",
    "        x  = F.relu(self.fc1(x))\n",
    "        x = F.tanh(self.fc2(x))\n",
    "        return x.cpu().data\n",
    "    \n",
    "    def evaluate(self, weights, gamma=1.0, max_t = 5000):\n",
    "        self.set_weights(weights)\n",
    "        episode_return = 0.0\n",
    "        state = env.reset()\n",
    "        for t in range(max_t):\n",
    "            state = torch.from_numpy(state).float().to(device)\n",
    "            action = self.forward(state)\n",
    "            state,reward,done,_ =env.step(action)\n",
    "            episode_return += reward*math.pow(gamma,t)\n",
    "            ## elegant implementation discounting the reward\n",
    "            if done:\n",
    "                break\n",
    "        return episode_return\n",
    "        \n",
    "agent = Agent(env).to(device)\n",
    "\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Train the Agent with a Cross-Entropy Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cem(n_iterations=500, max_t= 1000, gamma=1.0, print_every=100, pop_size=50, elite_frac=0.2, sigma=0.5):\n",
    "    \"\"\"PyTorch implementation of a cross-entropy method.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "        n_iterations (int): maximum number of training iterations\n",
    "        max_t (int): maximum number of timesteps per episode \n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        pop_size (int): size of population at each iteration\n",
    "        elite_frac (float): percentage of top performers to use in update\n",
    "        sigma (float): standard deviation of additive noise\n",
    "        \n",
    "    \"\"\"\n",
    "    n_elite = int(pop_size*elite_frac)\n",
    "    \n",
    "    scores_deque = deque(maxlen = 100)\n",
    "    scores= []\n",
    "    best_weight = sigma*np.random.randn(agent.get_weights_dim())\n",
    "    \n",
    "    for i_iteration in range(1, n_iterations+1):\n",
    "        \n",
    "        ## In below step we are training one complete episode for pop_size times (e.g 50 episodes and with different\n",
    "        ## initializing and collecting all cumulative rewards; 50 rewards in this case) \n",
    "        weights_pop = [best_weight + (sigma*np.random.randn(agent.get_weights_dim())) for i in range(pop_size)]\n",
    "        rewards = np.array([agent.evaluate(weights,gamma,max_t) for weights in weights_pop])\n",
    "        \n",
    "        ## Now getting the index(weight) of top 20% of model.\n",
    "        elite_idxs = rewards.argsort()[-n_elite:]\n",
    "        elite_weights = [weights_pop[i] for i in elite_idxs]\n",
    "        best_weight =np.array(elite_weights).mean(axis=0)\n",
    "        \n",
    "        reward = agent.evaluate(best_weight, gamma=1.0)\n",
    "        scores_deque.append(reward)\n",
    "        scores.append(reward)\n",
    "        \n",
    "        torch.save(agent.state_dict(), 'checkpoint.pth')\n",
    "        \n",
    "        if i_iteration%print_every==0:\n",
    "            print('Episode {}\\Average Score: {:.2f}'.format(i_iteration, np.mean(scores_deque)))\n",
    "            \n",
    "        if np.mean(scores_deque)>=90.0:\n",
    "            print('\\nEnvironment solved in {:d} iterations!\\tAverage Score: {:.2f}'.fromat(\n",
    "                i_iteration-100,np.mean(scores_deque)))\n",
    "            break\n",
    "    return scores\n",
    "\n",
    "scores =cem()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "drlnd",
   "language": "python",
   "name": "drlnd"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}