MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1)Loading input data\n",
    "We will use placeholder for this sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#download mnist dataset\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
    "\n",
    "#create a placeholder with two dimension\n",
    "#the first one is yet unknown the second one is the number of features in mnist i.e. 28x28 image\n",
    "x = tf.placeholder(tf.float32, [None,784])\n",
    "\n",
    "#create a placeholder for the labels\n",
    "y = tf.placeholder(tf.int64, [None,10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2)Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_linear_layer(x,num_output):\n",
    "    #creates a linear layer with num_output outputs\n",
    "    input_size = x.get_shape()[-1].value\n",
    "    W = tf.get_variable(\"weights\",shape=[input_size,num_output],initializer=tf.contrib.layers.xavier_initializer())  \n",
    "    b = tf.get_variable(\"biases\",shape=[num_output],initializer=tf.random_uniform_initializer(-0.5,0.5))\n",
    "    \n",
    "    return tf.matmul(x,W)+b\n",
    "\n",
    "def get_fc_layer(x,num_neurons):\n",
    "    #create a fully connected layer with num_neurons neurons\n",
    "    return tf.nn.relu(get_linear_layer(x,num_neurons))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Softmax Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#model prediction\n",
    "prediction_logits = get_linear_layer(x,10)\n",
    "prediction = tf.nn.softmax(prediction_logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2 layer neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope('hidden_layer'):\n",
    "    #fully connected layer 1\n",
    "    fc1 = get_fc_layer(x,50)\n",
    "\n",
    "with tf.variable_scope('output'):\n",
    "    #model prediction\n",
    "    prediction_logits = get_linear_layer(fc1,10)\n",
    "    prediction = tf.nn.softmax(prediction_logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3)Loss function definiton\n",
    "Both the model use cross entropy as loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=prediction_logits, name='cross_entropy'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4)Create minimizer and train op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")         #variable to keep track of the steps\n",
    "lr = 0.01                                                                 #learning rate\n",
    "\n",
    "train_op = tf.train.MomentumOptimizer(lr,0.90).minimize(loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also want to evaluate the performance of the learned model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(prediction,1))   #array of bool\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5) Create session and train model\n",
    "We will use an interactive session for this sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess=tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Iniatialize the variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train for 10000 step with batchsize 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i in range(10000):\n",
    "    batch_x, batch_y = mnist.train.next_batch(128)\n",
    "    _,loss_value = sess.run([train_op,loss], feed_dict={x: batch_x, y: batch_y})\n",
    "    if i%10 is 0:\n",
    "        print('Step {}/10000, loss: {}'.format(i,loss_value))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "acc_score = sess.run(accuracy, feed_dict={x:mnist.test.images, y:mnist.test.labels})\n",
    "print('Model accuracy after 1000 step: {}'.format(acc_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize an image with the prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "img_batch,pred = sess.run([x,prediction], feed_dict={x:batch_x,y:batch_y})\n",
    "img = img_batch[0]\n",
    "img = np.reshape(img,(28,28))\n",
    "plt.imshow(img, cmap='gray')\n",
    "print('Raw predictions: {}'.format(pred[0]))\n",
    "print('Class predicted {}'.format(np.argmax(pred[0])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Close Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}