UnsupervisedComponentAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Numerical tools\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "from scipy import signal\n",
    "from sklearn.decomposition import PCA, KernelPCA, FastICA\n",
    "\n",
    "# Viz\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PCA, kernel PCA, ICA, LDA\n",
    "\n",
    "This notebook is inspired by an interest in unsupervised data exploration. It might use some concept from the notebook on information theory.\n",
    "We also used this nice blog article: https://towardsdatascience.com/kernel-pca-vs-pca-vs-ica-in-tensorflow-sklearn-60e17eb15a64\n",
    "\n",
    "Interestingly another probabilistic approach is available through linear and quadratic discriminant analysis:\n",
    "https://www.datascienceblog.net/post/machine-learning/linear-discriminant-analysis/\n",
    "\n",
    "There is also something called probabilistic PCA: \n",
    "https://people.cs.pitt.edu/~milos/courses/cs3750-Fall2007/lectures/class17.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"800\"\n",
       "            height=\"600\"\n",
       "            src=\"doc/KPCA.pdf\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f48182fbb70>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import IFrame\n",
    "IFrame(\"doc/KPCA.pdf\", width=800, height=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"800\"\n",
       "            height=\"600\"\n",
       "            src=\"doc/PPCA.pdf\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7f47980aaf28>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import IFrame\n",
    "IFrame(\"doc/PPCA.pdf\", width=800, height=600)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sklearn.decomposition.PCA\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html\n",
    "### sklearn.decomposition.KernelPCA\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.KernelPCA.html\n",
    "### sklearn.decomposition.FastICA\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.FastICA.html\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example inspired by https://scikit-learn.org/stable/auto_examples/decomposition/plot_ica_blind_source_separation.html#sphx-glr-auto-examples-decomposition-plot-ica-blind-source-separation-py\n",
    "# #############################################################################\n",
    "# Generate sample data\n",
    "np.random.seed(0)\n",
    "n_samples = 2000\n",
    "time = np.linspace(0, 8, n_samples)\n",
    "\n",
    "s1 = np.sin(2 * time)  # Signal 1 : sinusoidal signal\n",
    "s2 = np.sign(np.sin(3 * time))  # Signal 2 : square signal\n",
    "s3 = signal.sawtooth(2 * np.pi * time)  # Signal 3: saw tooth signal\n",
    "\n",
    "S = np.c_[s1, s2, s3]\n",
    "S += 0.2 * np.random.normal(size=S.shape)  # Add noise\n",
    "\n",
    "S /= S.std(axis=0)  # Standardize data\n",
    "# Mix data\n",
    "A = np.array([[1, 1, 1], [0.5, 2, 1.0], [1.5, 1.0, 2.0]])  # Mixing matrix\n",
    "X = np.dot(S, A.T)  # Generate observations\n",
    "\n",
    "# Compute ICA\n",
    "ica = FastICA(n_components=3)\n",
    "S_ = ica.fit_transform(X)  # Reconstruct signals\n",
    "A_ = ica.mixing_  # Get estimated mixing matrix\n",
    "\n",
    "# We can `prove` that the ICA model applies by reverting the unmixing.\n",
    "assert np.allclose(X, np.dot(S_, A_.T) + ica.mean_)\n",
    "\n",
    "# For comparison, compute PCA\n",
    "pca = PCA(n_components=3)\n",
    "H = pca.fit_transform(X)  # Reconstruct signals based on orthogonal components\n",
    "\n",
    "# #############################################################################\n",
    "# Plot results\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "models = [X, S, S_, H]\n",
    "names = ['Observations (mixed signal)',\n",
    "         'True Sources',\n",
    "         'ICA recovered signals',\n",
    "         'PCA recovered signals']\n",
    "colors = ['red', 'steelblue', 'orange']\n",
    "\n",
    "for ii, (model, name) in enumerate(zip(models, names), 1):\n",
    "    plt.subplot(4, 1, ii)\n",
    "    plt.title(name)\n",
    "    for sig, color in zip(model.T, colors):\n",
    "        plt.plot(sig, color=color)\n",
    "\n",
    "plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.46)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}