Autoencoder with SVM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting mnist/train-images-idx3-ubyte.gz\n",
      "Extracting mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting mnist/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# %matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from sklearn import svm\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "mnist = input_data.read_data_sets(\"mnist/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clears up names and variables\n",
    "tf.reset_default_graph()\n",
    "\n",
    "inputs = tf.placeholder(tf.float32, shape=(None, 28, 28, 1), name='input')\n",
    "compare = tf.placeholder(tf.float32, shape=(None, 28, 28, 1), name='compare')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#encoder\n",
    "\n",
    "\n",
    "#convolution: input, number of filters, kenerl size, activation function, padding, name\n",
    "#maxpool: input, kernel size, stride length, name\n",
    "\n",
    "#output => 28x28x32\n",
    "convolution_1 = tf.layers.conv2d(inputs, 32, [3,3], activation=tf.nn.relu, padding='same', name='convolution_1')\n",
    "#output => 14x14x32\n",
    "maxpool_1 = tf.layers.max_pooling2d(convolution_1, [2,2], strides=2, name='maxpool_1')\n",
    "\n",
    "#output => 14x14x64\n",
    "convolution_2 = tf.layers.conv2d(maxpool_1, 64, [3,3], activation=tf.nn.relu, padding='same', name='convolution_2')\n",
    "\n",
    "#output => 14x14x32\n",
    "convolution_3 = tf.layers.conv2d(convolution_2, 32, [3,3], activation=tf.nn.relu, padding='same', name='convolution_3')\n",
    "#output => 7x7x32\n",
    "maxpool_2 = tf.layers.max_pooling2d(convolution_3, [2,2], strides=2, name='maxpool_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flatten image and feature extraction\n",
    "\n",
    "# 1568\n",
    "flatten = tf.layers.flatten(maxpool_2)\n",
    "# 128\n",
    "dense = tf.layers.dense(flatten, 128, activation=tf.nn.relu)\n",
    "# 1568\n",
    "unflatten = tf.layers.dense(dense, 1568, activation=tf.nn.relu)\n",
    "\n",
    "#output => 7x7x32\n",
    "reshaping = tf.reshape(unflatten, [-1, 7, 7, 32])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#decoder\n",
    "\n",
    "\n",
    "#output => 7x7x32\n",
    "convolution_4 = tf.layers.conv2d(reshaping, 32, [3,3], activation=tf.nn.relu, padding='same', name='convolution_4')\n",
    "#output => 14x14x32\n",
    "upsampling_1 = tf.image.resize_nearest_neighbor(convolution_4, [14,14], name='upsampling_1')\n",
    "\n",
    "#output => 14x14x64\n",
    "convolution_5 = tf.layers.conv2d(upsampling_1, 64, [3,3], activation=tf.nn.relu, padding='same', name='convolution_5')\n",
    "\n",
    "#output => 14x14x32\n",
    "convolution_6 = tf.layers.conv2d(convolution_5, 32, [3,3], activation=tf.nn.relu, padding='same', name='convolution_6')\n",
    "#output => 28x28x32\n",
    "upsampling_2 = tf.image.resize_nearest_neighbor(convolution_6 , [28,28], name='upsampling_2')\n",
    "\n",
    "#output => 28x28x1\n",
    "logits = tf.layers.conv2d(upsampling_2, 1, [3,3], activation=None, padding='same')\n",
    "\n",
    "#reconstructs the image\n",
    "decoder = tf.nn.sigmoid(logits, name='decoder')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculates cross entropy loss\n",
    "loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=compare, logits=logits)\n",
    "loss = tf.reduce_mean(loss)\n",
    "\n",
    "#define optimizer\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train the autoencoder\n",
    "\n",
    "def train_autoencoder():\n",
    "    epochs = 1\n",
    "    batches = 20\n",
    "    start1 = time.time()\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "    # start training\n",
    "    with tf.Session() as session:\n",
    "        session.run(init)\n",
    "\n",
    "        for i in range(epochs):\n",
    "            for j in range(mnist.train.num_examples // batches):\n",
    "                batch = mnist.train.next_batch(batches)\n",
    "                imgs = batch[0].reshape((-1, 28, 28, 1))\n",
    "                loss_val, _ = session.run([loss, optimizer], feed_dict={inputs: imgs, compare: imgs})\n",
    "\n",
    "            if i == (epochs - 1):\n",
    "                saver.save(session, 'model/final_model.ckpt')\n",
    "\n",
    "        end1 = time.time()\n",
    "        time1 = end1 - start1\n",
    "\n",
    "        return time1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train the svm\n",
    "\n",
    "def train_svm():   \n",
    "    new_saver = tf.train.Saver()\n",
    "    start2 = time.time()\n",
    "\n",
    "    with tf.Session() as session: \n",
    "        new_saver.restore(session, tf.train.latest_checkpoint('model'))\n",
    "\n",
    "        training_data = []\n",
    "        for i in range(mnist.train.num_examples):\n",
    "            img = mnist.train.images[i]\n",
    "            reconstructed = session.run(dense, feed_dict={inputs: img.reshape((1, 28, 28, 1))})\n",
    "            training_data.append(reconstructed[0])\n",
    "\n",
    "        classifier = svm.SVC(gamma='scale', decision_function_shape='ovo')\n",
    "        classifier.fit(training_data, np.argmax(mnist.train.labels, axis=1))\n",
    "\n",
    "        end2 = time.time()\n",
    "        time2 = end2 - start2\n",
    "        \n",
    "        return time2, classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run the full test of the model\n",
    "\n",
    "def run_model(classifier):\n",
    "    new_saver = tf.train.Saver()\n",
    "\n",
    "    with tf.Session() as session:\n",
    "        new_saver.restore(session, tf.train.latest_checkpoint('model'))\n",
    "\n",
    "        prediction = []\n",
    "        for i in range(mnist.test.num_examples): \n",
    "            img = mnist.test.images[i]\n",
    "            reconstructed = session.run(dense, feed_dict={inputs: img.reshape((1, 28, 28, 1))})\n",
    "            test = classifier.predict([reconstructed[0]])\n",
    "            prediction.append(test)\n",
    "\n",
    "        accuracy = accuracy_score(np.argmax(mnist.test.labels, axis=1), prediction)\n",
    "        \n",
    "        return accuracy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from model\\final_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from model\\final_model.ckpt\n",
      "Time: 85.87671518325806\n",
      "Accuracy: 0.988\n",
      "Test Run 1 Complete...\n",
      "\n",
      "\n",
      "Total Time: 85.87671518325806\n",
      "Average Time: 85.87671518325806\n",
      "Average Accuracy: 0.988\n"
     ]
    }
   ],
   "source": [
    "# run training and testing\n",
    "\n",
    "sum_time = 0\n",
    "sum_accuracy = 0\n",
    "count = 1\n",
    "iterations = 1\n",
    "\n",
    "for i in range(iterations):\n",
    "    time_1 = train_autoencoder()\n",
    "    time_2, classifier = train_svm()\n",
    "    accuracy = run_model(classifier)\n",
    "    \n",
    "    sum_time += time_1 + time_2\n",
    "    sum_accuracy += accuracy\n",
    "    print(\"Time: {}\".format(time_1 + time_2))\n",
    "    print(\"Accuracy: {}\".format(accuracy))\n",
    "    print(\"Test Run {} Complete...\\n\".format(count))\n",
    "    count += 1\n",
    "    \n",
    "print(\"\\nTotal Time: {}\".format(sum_time))\n",
    "print(\"Average Time: {}\".format(sum_time/iterations))\n",
    "print(\"Average Accuracy: {}\".format(sum_accuracy/iterations))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}