notebooks/buildmodels.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Programming Language Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.0 Download Watson Developer Cloud, import libraries, and load train/test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install watson_developer_cloud==2.1.0\n",
    "!pip install wget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import csv\n",
    "import json\n",
    "import wget\n",
    "import base64\n",
    "import operator\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from os import listdir\n",
    "from collections import *\n",
    "from os.path import isfile, join\n",
    "from watson_developer_cloud import NaturalLanguageClassifierV1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wget.download( 'https://github.com/IBM/programming-language-classifier/blob/master/data/githubtrainingdatacompressed.npz?raw=true' )\n",
    "wget.download( 'https://github.com/IBM/programming-language-classifier/blob/master/data/githubtestdatacompressed.npz?raw=true' )\n",
    "\n",
    "train_data = np.array(np.load(\"githubtrainingdatacompressed.npz\")['arr_0'])\n",
    "test_data = np.array(np.load(\"githubtestdatacompressed.npz\")['arr_0'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 A little more preprocessing\n",
    "Break the training data into separate dictionaries indexed by pl type, and map training data to a csv file for [Watson Natural Language Classifier](https://console.bluemix.net/services/natural-language-classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pls = {}\n",
    "for row in range(len(train_data)):\n",
    "    if train_data[row][1].decode() not in pls:\n",
    "        pls[train_data[row][1].decode()] = []\n",
    "    pls[train_data[row][1].decode()].append(train_data[row][0].decode())\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "CSV cannot exceed 1024 characters for column width and 15000 rows. So each piece of code is pushed into a Pandas dataframe in at most 1024 character chunks. Watson cannot take empty column values either, so those are removed, then the dataframe is converted into a csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d = []\n",
    "chunk = 1024\n",
    "\n",
    "for i in train_data:\n",
    "        for j in range(0,len(i[0]),chunk):\n",
    "            text = re.sub(' +',' ',\" \".join(re.split(r'[^\\w]', re.sub(re.compile(\"/\\*.*?\\*/\",re.DOTALL ) ,\"\" ,i[0][j:j+chunk].decode('utf-8')))))   \n",
    "            d.append({'text': text, 'pl': i[1].decode()})\n",
    "\n",
    "df = pd.DataFrame(d, columns = ['text', 'pl'])\n",
    "df['text'].replace(' ', np.nan, inplace=True)\n",
    "df = df.dropna()\n",
    "df.to_csv('trainingdata.csv', header=['text','pl'],index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.0 Naive Bayes Classifier\n",
    "\n",
    "Here we train a Naive Bayes Classifier\n",
    "for a light review on Naive Bayes look through the slides on GitHub\n",
    "for a thorough background on this topic (and many others in Machine Learning) \n",
    "check out Tom Mitchell's Carnegie Mellon course \n",
    "http://cc-web.isri.cmu.edu/CourseCast/Viewer/Default.aspx?id=a666b6e6-ad23-4fa3-96ce-ae50a42f45a3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bayes_train(pldict, samples):\n",
    "    plprobs = {}\n",
    "    counts = Counter()\n",
    "    for i in pldict:\n",
    "        plprobs[i] = float(len(pldict[i]))/samples\n",
    "        \n",
    "    plwordprobs = {}\n",
    "    plwordcounts = {}\n",
    "    for pl in pldict:\n",
    "        plwordprobs[pl] = {}\n",
    "        plwordcounts[pl] = 0\n",
    "    \n",
    "    for pl in pldict:\n",
    "        for i in pldict[pl]:\n",
    "            counts.update(filter(None, re.split(r'[^\\w]', re.sub(re.compile(\"/\\*.*?\\*/\",re.DOTALL ) ,\"\" ,i))))\n",
    "            for word in counts:\n",
    "                if word not in plwordprobs[pl]:\n",
    "                    plwordprobs[pl][word] = counts[word]\n",
    "                else:\n",
    "                    plwordprobs[pl][word] += counts[word]\n",
    "                plwordcounts[pl] += counts[word]\n",
    "            plwordcount = 0\n",
    "            counts = Counter()\n",
    "    for pl in plwordprobs:   \n",
    "        for word in plwordprobs[pl]:\n",
    "            plwordprobs[pl][word] = float(plwordprobs[pl][word])/plwordcounts[pl]\n",
    "        \n",
    "    \n",
    "    return plprobs, plwordprobs\n",
    "    \n",
    "plprobs, plwordprobs = bayes_train(pls, len(train_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking out the distribution of programming languages in our training set, and 10 of the most commonly used words of a particular language, try replacing 'sh' with other languages and observe the output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plprobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(plwordprobs['sh'].items(), key=operator.itemgetter(1) ,reverse=True)[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using the Naive Bayes Classifier to predict on the test set, again use the CMU course as a reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testbayes(testdata,plprob,plwordprob):\n",
    "    Ypred = []\n",
    "\n",
    "    for row in testdata:\n",
    "        testcounter = Counter()\n",
    "        testcounter.update(filter(None, re.split(r'[^\\w]', re.sub(re.compile(\"/\\*.*?\\*/\",re.DOTALL ) ,\"\" ,str(row[0])))))\n",
    "\n",
    "        prob = {}\n",
    "        for key in plprob:\n",
    "            prob[key] = 0\n",
    "        for key in prob:\n",
    "            for i in testcounter:\n",
    "                if i not in plwordprobs[key]:\n",
    "                    plwordprob[key][i] = 1e-4\n",
    "                else:\n",
    "                    plwordprob[key][i] += 1e-4\n",
    "                prob[key] += testcounter[i]*np.log(plwordprob[key][i])\n",
    "            prob[key] += np.log(plprob[key])\n",
    "        Ypred.append(max(prob.items(), key=operator.itemgetter(1))[0])\n",
    "    \n",
    "    return Ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = testbayes(test_data, plprobs, plwordprobs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.0 Create Classifier with Watson NLC and Evaluate Classification Accuracy\n",
    "\n",
    "Authenticate with Watson NLC, send it the training data csv, wait for it to finish its training phase, and compute the accuracy of both models. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# @hidden_cell\n",
    "\n",
    "natural_language_classifier = NaturalLanguageClassifierV1(\n",
    "    username=\"YOURUSERNAME\",\n",
    "    password=\"YOURPASSWORD\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Create Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "with open('trainingdata.csv', 'rb') as training_data:\n",
    "    print(json.dumps(natural_language_classifier.create_classifier(training_data=training_data, metadata='{\"name\": \"Programming Language Classifier\",\"language\": \"en\"}').get_result(), indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Add Classifier ID\n",
    "\n",
    "Copy/Paste your classifier_id into the variable below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_id = \"YOURCLASSIFIERID\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Monitor the status of your classifer be using the API below. Once the classifier's `status` is `Available`, proceed to the next cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "natural_language_classifier.get_classifier(classifier_id).get_result()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 Compute model accuracy\n",
    "Compute and compate model accuracy for the  Naive Bayes Classifier and the Watson NLC models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_my_accuracy(pred, testdata):\n",
    "    count = 0\n",
    "    for i in range(len(pred)):\n",
    "        if pred[i] == testdata[i][1].decode():\n",
    "            count += 1\n",
    "    return float(count)/len(pred)\n",
    "\n",
    "def compute_watson_accuracy(pred, testdata):\n",
    "    count = 0\n",
    "    for i in range(len(pred)):\n",
    "        if pred[i] == testdata[i][1].decode():\n",
    "            count += 1\n",
    "    return float(count)/len(pred)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "watsonpred = []\n",
    "for i in test_data:\n",
    "    x = natural_language_classifier.classify(classifier_id,re.sub(' +',' ',\" \".join(re.split(r'[^\\w]', re.sub(re.compile(\"/\\*.*?\\*/\",re.DOTALL ) ,\"\" ,i[0].decode()))))[0:1024]).get_result()\n",
    "    watsonpred.append(x['top_class'])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"My classifier's accuracy: \" + str(compute_my_accuracy(predictions, test_data)))\n",
    "print(\"Watson's accuracy: \" + str(compute_watson_accuracy(watsonpred, test_data)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}