tools/getdata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Programming Language Classification IBM Code Pattern - Build Data set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download pygithub, import libraries, and authenticate with your github credentials "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!pip install pygithub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import github\n",
    "import base64\n",
    "import numpy as np\n",
    "from github import GithubException\n",
    "\n",
    "#Create a GitHub personal access token at https://github.com/settings/tokens\n",
    "login = 'YOURLOGIN'\n",
    "token = 'YOURTOKEN'\n",
    "\n",
    "g = github.Github(login, token)\n",
    "repos = []\n",
    "org = g.get_organization('IBM')\n",
    "\n",
    "#We'll look at code only with the following extensions. You can try this with other\n",
    "#programming languages and for binary classification (2, not 8, classes).\n",
    "targetlangs = ['.go','.java','.js','.m','.py','.sh','.swift','.xml']\n",
    "\n",
    "#Get the list of repos at https://github.com/IBM.\n",
    "for repo in g.get_organization('IBM').get_repos():\n",
    "    repos.append(repo.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List repos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'ibm.github.io',\n",
       " u'BluePic',\n",
       " u'SparkMultiuserBenchmark',\n",
       " u'owplan',\n",
       " u'otc-slack-broker',\n",
       " u'otc-pagerduty-broker',\n",
       " u'otc-saucelabs-broker',\n",
       " u'node-nano-doc-updater',\n",
       " u'node-log4js-qradar-syslog-appender',\n",
       " u'node-swagger-ui-middleware',\n",
       " u'xlc-community-edition-dockerfile',\n",
       " u'ObjectizedTextInput',\n",
       " u'openfridge',\n",
       " u'CognitiveConcierge',\n",
       " u'message-catalog-manager',\n",
       " u'node-qradar-audit-logs-middleware',\n",
       " u'openchecks',\n",
       " u'swift-enterprise-demo',\n",
       " u'pyxcli',\n",
       " u'logmet-client-njs',\n",
       " u'flocker-driver',\n",
       " u'node-i18n-utilities',\n",
       " u'openwhisk-data-processing-message-hub',\n",
       " u'openwhisk-data-processing-cloudant',\n",
       " u'openwhisk-serverless-apis',\n",
       " u'serverless-home-automation',\n",
       " u'openwhisk-getting-started-template',\n",
       " u'air-traffic-control',\n",
       " u'openwhisk-action-trigger-rule',\n",
       " u'openwhisk-polyglot-development',\n",
       " u'container-service-gitlab-sample',\n",
       " u'PodcastDownloader',\n",
       " u'og',\n",
       " u'container-service-getting-started-wt',\n",
       " u'openwhisk-message-hub-trigger',\n",
       " u'openwhisk-cloudant-trigger',\n",
       " u'openwhisk-rest-api-trigger',\n",
       " u'watson-online-store',\n",
       " u'loopback-in-five',\n",
       " u'cf-icd-plugin',\n",
       " u'Scalable-WordPress-deployment-on-Kubernetes',\n",
       " u'Scalable-Cassandra-deployment-on-Kubernetes',\n",
       " u'swift-install',\n",
       " u'swift-kitura-helloworld',\n",
       " u'swift-sandbox',\n",
       " u'swift-bluemix-dev-cli',\n",
       " u'container-journey-template',\n",
       " u'dsx-twitter-auto-analysis',\n",
       " u'Kubernetes-container-service-GitLab-sample',\n",
       " u'elasticsearch-spark-recommender',\n",
       " u'todo-list-fabric',\n",
       " u'Java-MicroProfile-on-Kubernetes',\n",
       " u'ubiquity',\n",
       " u'ubiquity-k8s',\n",
       " u'ubiquity-docker-plugin',\n",
       " u'GameOn-Java-Microservices-on-Kubernetes',\n",
       " u'acme-freight',\n",
       " u'acme-freight-controller',\n",
       " u'acme-freight-erp',\n",
       " u'acme-freight-mobile',\n",
       " u'acme-freight-recommendation',\n",
       " u'acme-freight-webui',\n",
       " u'powerai-market-sentiment',\n",
       " u'OpenWhisk-on-Kubernetes',\n",
       " u'rocc-software',\n",
       " u'perfect-chisel',\n",
       " u'hdl-tools',\n",
       " u'websphere-api-connect',\n",
       " u'starcraft2-replay-analysis',\n",
       " u'personal-wealth-portfolio-mgt-bot',\n",
       " u'node-log4js-logmet-appender',\n",
       " u'janusgraph-utils',\n",
       " u'vr-speech-sandbox-vive',\n",
       " u'Hybrid-Cloud-Applications-and-Services',\n",
       " u'spring-boot-microservices-on-kubernetes',\n",
       " u'xlf-community-edition-dockerfile',\n",
       " u'ibm-cloud-devops',\n",
       " u'todo-list-fabricV1',\n",
       " u'watson-banking-chatbot',\n",
       " u'Microservices-deployment-with-PaaS-Containers-and-Serverless-Platforms',\n",
       " u'Spark-on-zOS',\n",
       " u'expressjs-openwhisk',\n",
       " u'tjbot-sports-buddy',\n",
       " u'Restful-APIs-using-zOS-Connect',\n",
       " u'zOS-Application-Discovery-and-Delivery-Intelligence',\n",
       " u'microservices-traffic-management-using-istio',\n",
       " u'Cloud-Native-Workloads-on-LinuxONE',\n",
       " u'spring-boot-continuous-delivery',\n",
       " u'watson-discovery-news-alerting',\n",
       " u'Microservices-with-JHipster-and-Spring-Boot',\n",
       " u'vr-speech-sandbox-cardboard',\n",
       " u'resilient-java-microservices-with-istio',\n",
       " u'Applications-on-CloudFoundry',\n",
       " u'unity-sdk',\n",
       " u'pino-couch',\n",
       " u'CognitiveIoT',\n",
       " u'Predictive-Market-Stress-Testing',\n",
       " u'spring-boot-continuous-delivery-admin',\n",
       " u'watson-multimedia-analyzer',\n",
       " u'pixiedust-traffic-analysis',\n",
       " u'charts',\n",
       " u'guide-conveyor-simulator',\n",
       " u'pixiedust-facebook-analysis',\n",
       " u'watson-discovery-news',\n",
       " u'ttyspy',\n",
       " u'Hackernews-NLU',\n",
       " u'k8s-service-accounts',\n",
       " u'watson-discovery-analyze-data-breaches',\n",
       " u'Istio-getting-started',\n",
       " u'hyperledger-fabric-on-linux-one',\n",
       " u'voice-of-the-customer',\n",
       " u'pytorch-seq2seq',\n",
       " u'alexa-skill-watson-conversation',\n",
       " u'banking-digitalization-using-hybrid-cloud-with-mainframes',\n",
       " u'increase-retail-revenues-leveraging-hybrid-cloud-on-mainframes',\n",
       " u'financial-risk-management-using-machine-learning-on-mainframes',\n",
       " u'rotisserie',\n",
       " u'chatbot-deployer',\n",
       " u'watson-document-classifier',\n",
       " u'powerai-transfer-learning',\n",
       " u'watson-conversation-slots-intro',\n",
       " u'IBM-Z-zOS',\n",
       " u'wcs-ocaml',\n",
       " u'liberty-outage-reporter',\n",
       " u'BlockchainNetwork-CompositeJourney',\n",
       " u'BlockchainSmartContractTrading-CompositeJourney',\n",
       " u'Decentralized-Energy-Composer',\n",
       " u'MaximoCOBieParser',\n",
       " u'MaximoForgeViewerPlugin',\n",
       " u'automated-monitoring-service-deployment-with-cuttle',\n",
       " u'serverless-twitter-insights',\n",
       " u'github-traffic-stats-collector',\n",
       " u'OpenWhiskChatAssistant',\n",
       " u'detect-timeseriesdata-change',\n",
       " u'node-red-dsx-workflow',\n",
       " u'openwhisk-mobile-backend',\n",
       " u'openwhisk-data-processing-cognitive',\n",
       " u'openwhisk-iot-ready',\n",
       " u'openwhisk-scheduled-tasks',\n",
       " u'openwhisk-conversational-scenarios',\n",
       " u'Integrate-Investment-Portfolio',\n",
       " u'graph-db-insights',\n",
       " u'deploy-graph-db-container',\n",
       " u'cognitive-social-crm',\n",
       " u'BlockchainPublicRegulationFabric-Food',\n",
       " u'SystemML_Usage',\n",
       " u'real-time-payments',\n",
       " u'watson-calorie-counter',\n",
       " u'tensorflow-hangul-recognition',\n",
       " u'sparksql-for-hbase',\n",
       " u'watson-document-co-relation',\n",
       " u'zos-node-accessor',\n",
       " u'tensorflow-kubernetes-art-classification',\n",
       " u'ibm-cos-sdk-java',\n",
       " u'ibm-cos-sdk-js',\n",
       " u'ibm-cos-sdk-python',\n",
       " u'ibm-cos-sdk-python-core',\n",
       " u'metrics-collector-service',\n",
       " u'metrics-collector-client-node',\n",
       " u'ibm-cos-sdk-python-s3transfer',\n",
       " u'spark-tpc-ds-performance-test',\n",
       " u'metrics-collector-client-python',\n",
       " u'watson-nao-robot',\n",
       " u'activator-lagom-java-chirper',\n",
       " u'deploy-ibm-cloud-private',\n",
       " u'microprofile-meeting',\n",
       " u'microprofile-meeting-persistence',\n",
       " u'microprofile-meeting-concurrency',\n",
       " u'microprofile-meeting-websockets',\n",
       " u'drupal-nginx-php-kubernetes',\n",
       " u'vue-a11y-calendar',\n",
       " u'metrics-collector-client-java',\n",
       " u'iot-predictive-analytics',\n",
       " u'gdpr-fingerprint-pii',\n",
       " u'lagraph',\n",
       " u'grid',\n",
       " u'grid-checker',\n",
       " u'metrics-collector-client-kubernetes',\n",
       " u'motion',\n",
       " u'java-microprofile-config',\n",
       " u'metrics-collector-client-swift',\n",
       " u'comanche',\n",
       " u'type',\n",
       " u'type-checker',\n",
       " u'kafka-streaming',\n",
       " u'open-source-service-lifecycle-mgmt',\n",
       " u'osslm-ansible-resource-manager',\n",
       " u'osslm-ucd-adaptor',\n",
       " u'manage-control-device-node-red',\n",
       " u'akka-react-cloudant',\n",
       " u'Portfolio-Optimize',\n",
       " u'vr-speech-sandbox-rift',\n",
       " u'cloud-journeys',\n",
       " u'authenticate-mobile-apps',\n",
       " u'hubot-cloudant-brain',\n",
       " u'watson-review-analyzer',\n",
       " u'watson-waste-sorter',\n",
       " u'Predictive-Industrial-Visual-Analysis',\n",
       " u'FoodTrackerBackend',\n",
       " u'ToDoBackend',\n",
       " u'image-analysis-iot-alert',\n",
       " u'build-blockchain-insurance-app',\n",
       " u'stop-malicious-http-traffic-with-Istio',\n",
       " u'osslm-docker-adaptor',\n",
       " u'watson-vehicle-damage-analyzer',\n",
       " u'Portfolio-compute',\n",
       " u'osslm-OpenSourceMANO-adaptor',\n",
       " u'watson-hamilton-lyrics-analysis',\n",
       " u'ibm-research-labs.github.io',\n",
       " u'powerai-collision-near-miss',\n",
       " u'engineering-insights']"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Directory\n",
    "\n",
    "This function recursively crawls through each file/directory starting at the given repo and path. \n",
    "It is similar in functionality to the \"find\" linux command\n",
    "Calls to GitHub API are wrapped in Try/Except blocks to ignore errors, there will be plenty of code to grab for our dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_directory(repository, path):\n",
    "    global dataset\n",
    "    try:\n",
    "        contents = repository.get_contents(path)\n",
    "        for content in contents:\n",
    "            if content.type == 'dir':\n",
    "                download_directory(repository, content.path)\n",
    "            else:\n",
    "                if content.content:\n",
    "                    if len(str(content.name).split(\".\")) == 2:\n",
    "                        if any(substring == (\".\" + str(content.name).split(\".\")[1]) for substring in targetlangs):                           \n",
    "                            try:\n",
    "                                dataset.append([base64.b64decode(content.content),str(content.name).split(\".\")[1]])\n",
    "                            except (GithubException, IOError) as exc:\n",
    "                                print 'Error processing %s: %s', content.path, exc\n",
    "    except (GithubException, IOError) as exc:\n",
    "        print \"error in dir \" \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "loop through each repo and grab the contents that match one of our target languages\n",
    "they are being places in a multidimensional list with \n",
    "dataset[i][0] being the code text and dataset[i][1] being the code's file extenstion\n",
    "\n",
    "\n",
    "there are two loops because I would ofter receive a 403 RateLimitExceededException from github\n",
    "\n",
    "if this happens just pick up in the next repo, again plenty of code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting repo 0 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 2 of 210\n",
      "getting repo 4 of 210\n",
      "getting repo 6 of 210\n",
      "getting repo 8 of 210\n",
      "getting repo 10 of 210\n",
      "error in dir \n",
      "getting repo 12 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 14 of 210\n",
      "getting repo 16 of 210\n",
      "getting repo 18 of 210\n",
      "getting repo 20 of 210\n",
      "getting repo 22 of 210\n",
      "getting repo 24 of 210\n",
      "getting repo 26 of 210\n",
      "getting repo 28 of 210\n",
      "getting repo 30 of 210\n",
      "getting repo 32 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 34 of 210\n",
      "getting repo 36 of 210\n",
      "getting repo 38 of 210\n",
      "getting repo 40 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 42 of 210\n"
     ]
    },
    {
     "ename": "RateLimitExceededException",
     "evalue": "403 {u'documentation_url': u'https://developer.github.com/v3/#rate-limiting', u'message': u'API rate limit exceeded for PubChimps.'}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRateLimitExceededException\u001b[0m                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-33-110917ec8fcb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0;34m\"getting repo \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\" of \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mrepo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_repo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0mdownload_directory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/nickacosta/anaconda/lib/python2.7/site-packages/github/Organization.pyc\u001b[0m in \u001b[0;36mget_repo\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    520\u001b[0m         headers, data = self._requester.requestJsonAndCheck(\n\u001b[1;32m    521\u001b[0m             \u001b[0;34m\"GET\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 522\u001b[0;31m             \u001b[0;34m\"/repos/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogin\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    523\u001b[0m         )\n\u001b[1;32m    524\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mgithub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRepository\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRepository\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_requester\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompleted\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/nickacosta/anaconda/lib/python2.7/site-packages/github/Requester.pyc\u001b[0m in \u001b[0;36mrequestJsonAndCheck\u001b[0;34m(self, verb, url, parameters, headers, input, cnx)\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrequestJsonAndCheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__check\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequestJson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mverb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrequestMultipartAndCheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/nickacosta/anaconda/lib/python2.7/site-packages/github/Requester.pyc\u001b[0m in \u001b[0;36m__check\u001b[0;34m(self, status, responseHeaders, output)\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__structuredFromJson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m400\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__createException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponseHeaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponseHeaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRateLimitExceededException\u001b[0m: 403 {u'documentation_url': u'https://developer.github.com/v3/#rate-limiting', u'message': u'API rate limit exceeded for PubChimps.'}"
     ]
    }
   ],
   "source": [
    "dataset = []\n",
    "\n",
    "for i in range(len(repos[1:])):\n",
    "    if i % 2 == 0:\n",
    "        print \"getting repo \" + str(i) + \" of \" + str(len(repos[1:]))\n",
    "    repo = org.get_repo(str(repos[i]))\n",
    "    download_directory(repo, \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting repo 44 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 46 of 210\n",
      "getting repo 48 of 210\n",
      "getting repo 50 of 210\n",
      "getting repo 52 of 210\n",
      "getting repo 54 of 210\n",
      "error in dir \n",
      "getting repo 56 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 58 of 210\n",
      "getting repo 60 of 210\n",
      "getting repo 62 of 210\n",
      "getting repo 64 of 210\n",
      "getting repo 66 of 210\n",
      "getting repo 68 of 210\n",
      "getting repo 70 of 210\n",
      "getting repo 72 of 210\n",
      "getting repo 74 of 210\n",
      "getting repo 76 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 78 of 210\n",
      "getting repo 80 of 210\n",
      "getting repo 82 of 210\n",
      "getting repo 84 of 210\n",
      "error in dir \n",
      "getting repo 86 of 210\n",
      "getting repo 88 of 210\n",
      "getting repo 90 of 210\n",
      "error in dir \n",
      "getting repo 92 of 210\n",
      "getting repo 94 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 96 of 210\n",
      "getting repo 98 of 210\n",
      "getting repo 100 of 210\n",
      "getting repo 102 of 210\n",
      "error in dir \n",
      "getting repo 104 of 210\n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "error in dir \n",
      "getting repo 106 of 210\n"
     ]
    },
    {
     "ename": "RateLimitExceededException",
     "evalue": "403 {u'documentation_url': u'https://developer.github.com/v3/#rate-limiting', u'message': u'API rate limit exceeded for PubChimps.'}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRateLimitExceededException\u001b[0m                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-131-e9d1d85946a0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0;34m\"getting repo \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m44\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\" of \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mrepo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_repo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mdownload_directory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/nickacosta/anaconda/lib/python2.7/site-packages/github/Organization.pyc\u001b[0m in \u001b[0;36mget_repo\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    520\u001b[0m         headers, data = self._requester.requestJsonAndCheck(\n\u001b[1;32m    521\u001b[0m             \u001b[0;34m\"GET\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 522\u001b[0;31m             \u001b[0;34m\"/repos/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogin\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    523\u001b[0m         )\n\u001b[1;32m    524\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mgithub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRepository\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRepository\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_requester\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompleted\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/nickacosta/anaconda/lib/python2.7/site-packages/github/Requester.pyc\u001b[0m in \u001b[0;36mrequestJsonAndCheck\u001b[0;34m(self, verb, url, parameters, headers, input, cnx)\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrequestJsonAndCheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__check\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequestJson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mverb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrequestMultipartAndCheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/nickacosta/anaconda/lib/python2.7/site-packages/github/Requester.pyc\u001b[0m in \u001b[0;36m__check\u001b[0;34m(self, status, responseHeaders, output)\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__structuredFromJson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m400\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__createException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponseHeaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponseHeaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRateLimitExceededException\u001b[0m: 403 {u'documentation_url': u'https://developer.github.com/v3/#rate-limiting', u'message': u'API rate limit exceeded for PubChimps.'}"
     ]
    }
   ],
   "source": [
    "for i in range(len(repos[44:])):\n",
    "    if i % 2 == 0:\n",
    "        print \"getting repo \" + str(i+44) + \" of \" + str(len(repos[1:]))\n",
    "    repo = org.get_repo(str(repos[i]))\n",
    "    download_directory(repo, \"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the code into a training and a testing set.\n",
    "You can see the length of the dataset and its contents.\n",
    "Pickle the training and testing sets for the next notebook (mine are included in the github)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "data = dataset\n",
    "random.shuffle(data)\n",
    "train_data = data[:int((len(data)+1)*.80)] #Remaining 80% to training set\n",
    "test_data = data[int(len(data)*.80+1):] #Splits 20% data to test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2300"
      ]
     },
     "execution_count": 452,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "py\n"
     ]
    }
   ],
   "source": [
    "print dataset[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez_compressed(\"githubtrainingdatacompressed.npz\",np.array(train_data))\n",
    "np.savez_compressed(\"githubtestdatacompressed.npz\",np.array(test_data))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}