cnn_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf\n",
    "\n",
    "# Download MNIST data\n",
    "mnist = input_data.read_data_sets('/tmp/data/', one_hot=True)\n",
    "sess = tf.InteractiveSession()\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "x_image = tf.reshape(x, [-1,28,28,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_mnist(y, iterations=1000, batch_size=100):\n",
    "    # Define loss and optimizer\n",
    "    cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))\n",
    "    train_step = tf.train.AdamOptimizer().minimize(cross_entropy)\n",
    "\n",
    "    # Train the model\n",
    "    tf.initialize_all_variables().run()\n",
    "    for i in range(iterations):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        train_step.run({x: batch_xs, y_: batch_ys})\n",
    "\n",
    "    # Test the model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    validation_accuracy = accuracy.eval({x: mnist.validation.images, y_: mnist.validation.labels})\n",
    "    test_accuracy = accuracy.eval({x: mnist.test.images, y_: mnist.test.labels})\n",
    "    return validation_accuracy, test_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Grid search for hyper parameters\n",
    "import itertools\n",
    "import sys\n",
    "\n",
    "def optimize_mnist(get_model, *hyperparameters):\n",
    "    print('validation, test, hyperparameter')\n",
    "    best = None\n",
    "    \n",
    "    for hyperparameter in itertools.product(*hyperparameters):\n",
    "        model = get_model(*hyperparameter)\n",
    "        validation_accuracy, test_accuracy = train_mnist(model)\n",
    "        print(validation_accuracy, test_accuracy, hyperparameter)\n",
    "        sys.stdout.flush()\n",
    "        \n",
    "        if best is None or validation_accuracy > best[0]:\n",
    "            best = (validation_accuracy, test_accuracy, hyperparameter)\n",
    "    print('best setting')\n",
    "    print(*best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.98479998, 0.98710001)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# \"Deep MNIST for Experts\" tutorial\n",
    "stddev = 0.01\n",
    "b_init = 0.01\n",
    "\n",
    "W_conv1 = tf.Variable(tf.truncated_normal([5, 5, 1, 32], stddev=stddev))\n",
    "b_conv1 = tf.Variable(tf.constant(b_init, shape=[32]))\n",
    "o_conv1 = tf.nn.conv2d(x_image, W_conv1, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "h_conv1 = tf.nn.relu(o_conv1 + b_conv1)\n",
    "h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 32, 64], stddev=stddev))\n",
    "b_conv2 = tf.Variable(tf.constant(b_init, shape=[64]))\n",
    "o_conv2 = tf.nn.conv2d(h_pool1, W_conv2, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "h_conv2 = tf.nn.relu(o_conv2 + b_conv2)\n",
    "h_pool2 = tf.nn.max_pool(h_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "W_fc1 = tf.Variable(tf.truncated_normal([7 * 7 * 64, 1024], stddev=stddev))\n",
    "b_fc1 = tf.Variable(tf.constant(b_init, shape=[1024]))\n",
    "\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "W_fc2 = tf.Variable(tf.truncated_normal([1024, 10], stddev=stddev))\n",
    "b_fc2 = tf.Variable(tf.constant(b_init, shape=[10]))\n",
    "\n",
    "y = tf.nn.softmax(tf.matmul(h_fc1, W_fc2) + b_fc2)\n",
    "\n",
    "train_mnist(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation, test, hyperparameter\n",
      "0.9726 0.9766 (16, 5, 1, 1)\n",
      "0.968 0.9711 (16, 5, 1, 2)\n",
      "0.9614 0.96 (16, 5, 1, 4)\n",
      "0.9808 0.9813 (16, 10, 1, 1)\n",
      "0.9786 0.9811 (16, 10, 1, 2)\n",
      "0.974 0.9767 (16, 10, 1, 4)\n",
      "0.986 0.986 (16, 20, 1, 1)\n",
      "0.9832 0.9826 (16, 20, 1, 2)\n",
      "0.9802 0.9811 (16, 20, 1, 4)\n",
      "0.9842 0.9819 (16, 27, 1, 1)\n",
      "0.9816 0.9836 (16, 27, 1, 2)\n",
      "0.9798 0.9792 (16, 27, 1, 4)\n",
      "0.9766 0.9788 (32, 5, 1, 1)\n",
      "0.9766 0.9771 (32, 5, 1, 2)\n",
      "0.9724 0.9734 (32, 5, 1, 4)\n",
      "0.9844 0.9811 (32, 10, 1, 1)\n",
      "0.9796 0.9799 (32, 10, 1, 2)\n",
      "0.9788 0.9776 (32, 10, 1, 4)\n",
      "0.9844 0.9861 (32, 20, 1, 1)\n",
      "0.9852 0.9852 (32, 20, 1, 2)\n",
      "0.986 0.9822 (32, 20, 1, 4)\n",
      "0.9836 0.9845 (32, 27, 1, 1)\n",
      "0.9848 0.9864 (32, 27, 1, 2)\n",
      "0.9838 0.9837 (32, 27, 1, 4)\n",
      "0.9806 0.9815 (64, 5, 1, 1)\n",
      "0.9786 0.9818 (64, 5, 1, 2)\n",
      "0.9766 0.9766 (64, 5, 1, 4)\n",
      "0.9836 0.9851 (64, 10, 1, 1)\n",
      "0.9858 0.9855 (64, 10, 1, 2)\n",
      "0.9816 0.985 (64, 10, 1, 4)\n",
      "0.9818 0.9817 (64, 20, 1, 1)\n",
      "0.9842 0.9861 (64, 20, 1, 2)\n",
      "0.9864 0.9873 (64, 20, 1, 4)\n",
      "0.9846 0.9855 (64, 27, 1, 1)\n",
      "0.9834 0.9831 (64, 27, 1, 2)\n",
      "0.9822 0.9829 (64, 27, 1, 4)\n",
      "best setting\n",
      "0.9864 0.9873 (64, 20, 1, 4)\n"
     ]
    }
   ],
   "source": [
    "# Simple convolution + softmax layer with various settings\n",
    "stddev = 0.01\n",
    "b_init = 0.01\n",
    "\n",
    "def simple_cnn(channel, patch, stride, pool):\n",
    "    W_conv = tf.Variable(tf.truncated_normal([patch, patch, 1, channel], stddev=stddev))\n",
    "    b_conv = tf.Variable(tf.constant(b_init, shape=[channel]))\n",
    "    o_conv = tf.nn.conv2d(x_image, W_conv, strides=[1, stride, stride, 1], padding='SAME')\n",
    "    h_conv = tf.nn.relu(o_conv + b_conv)\n",
    "    h_pool = tf.nn.max_pool(h_conv, ksize=[1, pool, pool, 1], strides=[1, pool, pool, 1], padding='SAME')\n",
    "    h_size = int(28 / stride / pool) ** 2 * channel\n",
    "    h_pool_flat = tf.reshape(h_pool, [-1, h_size])\n",
    "\n",
    "    W_fc = tf.Variable(tf.truncated_normal([h_size, 10], stddev=stddev))\n",
    "    b_fc = tf.Variable(tf.constant(b_init, shape=[10]))\n",
    "\n",
    "    y = tf.nn.softmax(tf.matmul(h_pool_flat, W_fc) + b_fc)\n",
    "    return y\n",
    "\n",
    "channel = [16, 32, 64]\n",
    "patch = [5, 10, 20, 27]\n",
    "stride = [1]\n",
    "pool = [1, 2, 4]\n",
    "optimize_mnist(simple_cnn, channel, patch, stride, pool)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}