{
 "metadata": {
  "name": "v1-dev"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Version 1 is the same neural network as on Coursera's machine learning class. 1 Hidden Layer."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "from scipy import optimize\n",
      "from scipy.io import loadmat\n",
      "from __future__ import division"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "num_features = 5000\n",
      "input_layer_size  = 400\n",
      "hidden_layer_size = 25\n",
      "num_labels = 10"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Load data / trained thetas"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "data = loadmat('ex3data1.mat')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, y = data['X'], data['y']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "y = y.reshape(X.shape[0], )"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "y = y - 1  # Fix notation"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X.shape, y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "((5000, 400), (5000,))"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "data = loadmat('ex3weights.mat')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "theta1_sol, theta2_sol = data['Theta1'], data['Theta2']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "theta1_sol.shape, theta2_sol.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "((25, 401), (10, 26))"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Activation function and derivative"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def activation_func(z):\n",
      "    return 1 / (1 + np.exp(-z))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def activiation_func_prime(z):\n",
      "    sig = activation_func(z)\n",
      "    return sig * (1 - sig)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Forward / Predict: Using the solution"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def _forward(X, T1, T2):\n",
      "    m = X.shape[0]\n",
      "    ones = None\n",
      "    if len(X.shape) == 1:\n",
      "        ones = np.array(1).reshape(1,)\n",
      "    else:\n",
      "        ones = np.ones(m).reshape(m,1)\n",
      "    \n",
      "    # Input layer\n",
      "    a1 = np.hstack((ones, X))  # [5000, 401]\n",
      "    \n",
      "    # Hidden Layer\n",
      "    z2 = np.dot(T1, a1.T)  # [25, 5000]\n",
      "    a2 = activation_func(z2)  # [25, 5000]\n",
      "    a2 = np.hstack((ones, a2.T))  # [26, 5000]\n",
      "    \n",
      "    # Output layer\n",
      "    z3 = np.dot(T2, a2.T)  # [10, 5000]\n",
      "    a3 = activation_func(z3)  # [10, 5000]\n",
      "    \n",
      "    return a3"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 136
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "_forward(X, theta1_sol, theta2_sol).shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 137,
       "text": [
        "(10, 5000)"
       ]
      }
     ],
     "prompt_number": 137
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "_forward(X[0], theta1_sol, theta2_sol).shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 138,
       "text": [
        "(10,)"
       ]
      }
     ],
     "prompt_number": 138
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def predict(X, t1, t2):\n",
      "    # _forward() contains the probalities, so take the max of each column\n",
      "    return _forward(X, t1, t2).argmax(0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 126
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(predict(X, theta1_sol, theta2_sol) == y).mean()  # > 0.95"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 127,
       "text": [
        "0.97519999999999996"
       ]
      }
     ],
     "prompt_number": 127
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "cost func"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def sumsqr(a):\n",
      "    return np.sum(a ** 2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 128
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def cost(X, y, t1, t2, reg_lambda):\n",
      "    m = X.shape[0]\n",
      "    Y = np.eye(num_labels)[y]  # [5000, 10]\n",
      "    \n",
      "    h = _forward(X, t1, t2)\n",
      "    costPositive = -Y * np.log(h).T\n",
      "    costNegative = (1 - Y) * np.log(1 - h).T\n",
      "    cost = costPositive - costNegative\n",
      "    J = np.sum(cost) / m\n",
      "    \n",
      "    t1f = t1[:, 1:]\n",
      "    t2f = t2[:, 1:]\n",
      "    reg = (reg_lambda / (2 * m)) * (sumsqr(t1f) + sumsqr(t2f))\n",
      "    J = J + reg\n",
      "    return J"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 129
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cost(X, y, theta1_sol, theta2_sol, 3)  # With the solution: lambda = 1 -> 0.38 | lambda = 3 -> 0.57"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 130,
       "text": [
        "0.57605124695013399"
       ]
      }
     ],
     "prompt_number": 130
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Random initialization"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def rand_init(l_in, l_out):\n",
      "    epsilon_init = 0.12\n",
      "    return np.random.rand(l_out, l_in + 1) * 2 * epsilon_init - epsilon_init"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "theta1_0 = rand_init(input_layer_size, hidden_layer_size)\n",
      "theta2_0 = rand_init(hidden_layer_size, num_labels)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "theta1_sol.shape, theta1_0.shape, theta2_sol.shape, theta2_0.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "((25, 401), (25, 401), (10, 26), (10, 26))"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Backprop"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "_t1f = theta1_0[:, 1:]\n",
      "_t2f = theta2_0[:, 1:]\n",
      "Y = np.eye(num_labels)[y]\n",
      "Delta1, Delta2 = 0, 0\n",
      "for i, row in enumerate(X):\n",
      "    a1 = np.append([1], row, axis=1)\n",
      "    z2 = np.dot(theta1_0, a1)\n",
      "    a2 = np.append([1], activation_func(z2), axis=1)\n",
      "    z3 = np.dot(theta2_0, a2)\n",
      "    a3 = activation_func(z3)\n",
      "    \n",
      "    # Step 2\n",
      "    d3 = a3 - Y[i, :].T\n",
      "\n",
      "\t# Step 3\n",
      "    d2 = np.dot(_t2f.T, d3) * activiation_func_prime(z2)\n",
      "    \n",
      "\t# Step 4\n",
      "    Delta2 = np.dot(d3[np.newaxis].T, a2[np.newaxis])  # [10, 26]\n",
      "    Delta1 = np.dot(d2[np.newaxis].T, a1[np.newaxis])  # [25, 401]\n",
      "\n",
      "Theta1_grad = (1 / X.shape[0]) * Delta1  # [25, 401]\n",
      "Theta2_grad = (1 / X.shape[0]) * Delta2  # [10, 26]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Theta1_grad.shape, theta1_sol.shape, Theta2_grad.shape, theta2_sol.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "((25, 401), (25, 401), (10, 26), (10, 26))"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Join everything & optimization"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def cost_func(thetas, input_layer_size, hidden_layer_size, num_labels, X, y, reg_lambda):\n",
      "    t1_start = 0\n",
      "    t1_end = hidden_layer_size * (input_layer_size + 1)\n",
      "    t1 = thetas[t1_start:t1_end].reshape((hidden_layer_size, input_layer_size + 1))\n",
      "    t2 = thetas[t1_end:].reshape((num_labels, hidden_layer_size + 1))\n",
      "    \n",
      "    # forward/cost\n",
      "    J = cost(X, y, t1, t2, reg_lambda)\n",
      "    \n",
      "    # backprop\n",
      "    m = X.shape[0] \n",
      "    t1f = t1[:, 1:]\n",
      "    t2f = t2[:, 1:]\n",
      "    Y = np.eye(num_labels)[y]\n",
      "    \n",
      "    Delta1, Delta2 = 0, 0\n",
      "    for i, row in enumerate(X):\n",
      "        a1 = np.append([1], row, axis=1)\n",
      "        z2 = np.dot(t1, a1)\n",
      "        a2 = np.append([1], activation_func(z2), axis=1)\n",
      "        z3 = np.dot(t2, a2)\n",
      "        a3 = activation_func(z3)\n",
      "        \n",
      "        # Step 2\n",
      "        d3 = a3 - Y[i, :].T\n",
      "        \n",
      "        # Step 3\n",
      "        d2 = np.dot(t2f.T, d3) * activiation_func_prime(z2)\n",
      "        \n",
      "        # Step 4\n",
      "        Delta2 += np.dot(d3[np.newaxis].T, a2[np.newaxis])  # [10, 26]\n",
      "        Delta1 += np.dot(d2[np.newaxis].T, a1[np.newaxis])  # [25, 401]\n",
      "        \n",
      "    Theta1_grad = (1 / m) * Delta1  # [25, 401]\n",
      "    Theta2_grad = (1 / m) * Delta2  # [10, 26]\n",
      "    \n",
      "    Theta1_grad[:, 1:] = Theta1_grad[:, 1:] + (reg_lambda / m) * t1f\n",
      "    Theta2_grad[:, 1:] = Theta2_grad[:, 1:] + (reg_lambda / m) * t2f\n",
      "    \n",
      "    return J, np.concatenate((Theta1_grad.reshape(-1), Theta2_grad.reshape(-1)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "thetas0 = np.concatenate((theta1_0.reshape(-1), theta2_0.reshape(-1)))\n",
      "cost_func(thetas0, input_layer_size, hidden_layer_size, num_labels, X, y, 0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "(7.0631025218626604,\n",
        " array([ 0.00728744,  0.        ,  0.        , ...,  0.17291724,\n",
        "        0.19521494,  0.19591867]))"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "_thetas, _, _ = optimize.fmin_tnc(cost_func, thetas0, args=(input_layer_size, hidden_layer_size, num_labels, X, y, 0))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-c:7: RuntimeWarning: divide by zero encountered in log\n",
        "-c:7: RuntimeWarning: invalid value encountered in multiply\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "t1_start = 0\n",
      "t1_end = hidden_layer_size * (input_layer_size + 1)\n",
      "_t1 = _thetas[t1_start:t1_end].reshape((hidden_layer_size, input_layer_size + 1))\n",
      "_t2 = _thetas[t1_end:].reshape((num_labels, hidden_layer_size + 1))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "predict(X, _t1, _t2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "array([9, 9, 9, ..., 8, 8, 8])"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(predict(X, _t1, _t2) == y).mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "1.0"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "fmin_cg"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def function(thetas, input_layer_size, hidden_layer_size, num_labels, X, y, reg_lambda):\n",
      "    t1_start = 0\n",
      "    t1_end = hidden_layer_size * (input_layer_size + 1)\n",
      "    t1 = thetas[t1_start:t1_end].reshape((hidden_layer_size, input_layer_size + 1))\n",
      "    t2 = thetas[t1_end:].reshape((num_labels, hidden_layer_size + 1))\n",
      "    \n",
      "    Y = np.eye(num_labels)[y]\n",
      "    J = cost(X, y, t1, t2, reg_lambda)\n",
      "    return J"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "function(thetas0, input_layer_size, hidden_layer_size, num_labels, X, y, 0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 36,
       "text": [
        "7.0631025218626604"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def function_prime(thetas, input_layer_size, hidden_layer_size, num_labels, X, y, reg_lambda):\n",
      "    t1_start = 0\n",
      "    t1_end = hidden_layer_size * (input_layer_size + 1)\n",
      "    t1 = thetas[t1_start:t1_end].reshape((hidden_layer_size, input_layer_size + 1))\n",
      "    t2 = thetas[t1_end:].reshape((num_labels, hidden_layer_size + 1))\n",
      "    \n",
      "    m = X.shape[0]\n",
      "    t1f = t1[:, 1:]\n",
      "    t2f = t2[:, 1:]\n",
      "    Y = np.eye(num_labels)[y]\n",
      "    \n",
      "    Delta1, Delta2 = 0, 0\n",
      "    for i, row in enumerate(X):\n",
      "        a1 = np.append([1], row, axis=1)\n",
      "        z2 = np.dot(t1, a1)\n",
      "        a2 = np.append([1], activation_func(z2), axis=1)\n",
      "        z3 = np.dot(t2, a2)\n",
      "        a3 = activation_func(z3)\n",
      "        \n",
      "        # Step 2\n",
      "        d3 = a3 - Y[i, :].T\n",
      "        \n",
      "        # Step 3\n",
      "        d2 = np.dot(t2f.T, d3) * activiation_func_prime(z2)\n",
      "        \n",
      "        # Step 4\n",
      "        Delta2 += np.dot(d3[np.newaxis].T, a2[np.newaxis])  # [10, 26]\n",
      "        Delta1 += np.dot(d2[np.newaxis].T, a1[np.newaxis])  # [25, 401]\n",
      "        \n",
      "    Theta1_grad = (1 / m) * Delta1  # [25, 401]\n",
      "    Theta2_grad = (1 / m) * Delta2  # [10, 26]\n",
      "    \n",
      "    Theta1_grad[:, 1:] = Theta1_grad[:, 1:] + (reg_lambda / m) * t1f\n",
      "    Theta2_grad[:, 1:] = Theta2_grad[:, 1:] + (reg_lambda / m) * t2f\n",
      "    \n",
      "    return np.concatenate((Theta1_grad.reshape(-1), Theta2_grad.reshape(-1)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 39
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "function_prime(thetas0, input_layer_size, hidden_layer_size, num_labels, X, y, 0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 40,
       "text": [
        "array([ 0.00728744,  0.        ,  0.        , ...,  0.17291724,\n",
        "        0.19521494,  0.19591867])"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "_thetas = optimize.fmin_cg(function, thetas0, fprime=function_prime, args=(input_layer_size, hidden_layer_size, num_labels, X, y, 0), maxiter=100)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Warning: Maximum number of iterations has been exceeded.\n",
        "         Current function value: 0.084507\n",
        "         Iterations: 100\n",
        "         Function evaluations: 251\n",
        "         Gradient evaluations: 251\n"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "t1_start = 0\n",
      "t1_end = hidden_layer_size * (input_layer_size + 1)\n",
      "_t1 = _thetas[t1_start:t1_end].reshape((hidden_layer_size, input_layer_size + 1))\n",
      "_t2 = _thetas[t1_end:].reshape((num_labels, hidden_layer_size + 1))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 42
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(predict(X, _t1, _t2) == y).mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 43,
       "text": [
        "0.99560000000000004"
       ]
      }
     ],
     "prompt_number": 43
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "With minimize (general)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "options = {'maxiter': 100}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 44
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "_res = optimize.minimize(function, thetas0, jac=function_prime, method='CG', args=(input_layer_size, hidden_layer_size, num_labels, X, y, 0), options=options)\n",
      "_thetas = _res.x"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 45
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "t1_start = 0\n",
      "t1_end = hidden_layer_size * (input_layer_size + 1)\n",
      "_t1 = _thetas[t1_start:t1_end].reshape((hidden_layer_size, input_layer_size + 1))\n",
      "_t2 = _thetas[t1_end:].reshape((num_labels, hidden_layer_size + 1))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(predict(X, _t1, _t2) == y).mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 47,
       "text": [
        "0.99560000000000004"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Benchmarks?"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%timeit optimize.minimize(function, thetas0, jac=function_prime, method='TNC', args=(input_layer_size, hidden_layer_size, num_labels, X, y, 0), options=options)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1 loops, best of 3: 98.7 s per loop\n"
       ]
      }
     ],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%timeit optimize.minimize(function, thetas0, jac=function_prime, method='CG', args=(input_layer_size, hidden_layer_size, num_labels, X, y, 0), options=options)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1 loops, best of 3: 257 s per loop\n"
       ]
      }
     ],
     "prompt_number": 49
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%timeit optimize.minimize(function, thetas0, jac=function_prime, method='BFGS', args=(input_layer_size, hidden_layer_size, num_labels, X, y, 0), options=options)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%timeit optimize.minimize(function, thetas0, jac=function_prime, method='L-BFGS-B', args=(input_layer_size, hidden_layer_size, num_labels, X, y, 0), options=options)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}