{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "1"
    },
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## AlexNet\n",
    "\n",
    "Use it on scaled-up 224x224 dimensional Fashion-MNIST with 10 class output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import d2l\n",
    "from mxnet import autograd, gluon, init, nd\n",
    "from mxnet.gluon import data as gdata, nn\n",
    "from mxnet.gluon import loss as gloss\n",
    "import os\n",
    "import sys\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "net = nn.Sequential()\n",
    "# Use a larger 11 x 11 window to capture objects. Stride of 4 to reduce size.\n",
    "# The number of output channels is much larger than that in LeNet.\n",
    "net.add(nn.Conv2D(96, kernel_size=11, strides=4, activation='relu'),\n",
    "        nn.MaxPool2D(pool_size=3, strides=2),\n",
    "        # Make the convolution window smaller, set padding to 2 for consistent height/width\n",
    "        nn.Conv2D(256, kernel_size=5, padding=2, activation='relu'),\n",
    "        nn.MaxPool2D(pool_size=3, strides=2),\n",
    "        # Use three successive convolutional layers and a smaller convolution window. \n",
    "        nn.Conv2D(384, kernel_size=3, padding=1, activation='relu'),\n",
    "        nn.Conv2D(384, kernel_size=3, padding=1, activation='relu'),\n",
    "        nn.Conv2D(256, kernel_size=3, padding=1, activation='relu'),\n",
    "        # Reduce dimensionality\n",
    "        nn.MaxPool2D(pool_size=3, strides=2),\n",
    "        # Expensive dense layer\n",
    "        nn.Dense(4096, activation=\"relu\"), nn.Dropout(0.5),\n",
    "        nn.Dense(4096, activation=\"relu\"), nn.Dropout(0.5),\n",
    "        # Output layer. Since we are using Fashion-MNIST, n = 10\n",
    "        nn.Dense(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "We construct a single-channel data instance with both height and width of 224 to observe the output shape of each layer. It matches our diagram above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "2"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv0 output shape:\t (1, 96, 54, 54)\n",
      "pool0 output shape:\t (1, 96, 26, 26)\n",
      "conv1 output shape:\t (1, 256, 26, 26)\n",
      "pool1 output shape:\t (1, 256, 12, 12)\n",
      "conv2 output shape:\t (1, 384, 12, 12)\n",
      "conv3 output shape:\t (1, 384, 12, 12)\n",
      "conv4 output shape:\t (1, 256, 12, 12)\n",
      "pool2 output shape:\t (1, 256, 5, 5)\n",
      "dense0 output shape:\t (1, 4096)\n",
      "dropout0 output shape:\t (1, 4096)\n",
      "dense1 output shape:\t (1, 4096)\n",
      "dropout1 output shape:\t (1, 4096)\n",
      "dense2 output shape:\t (1, 10)\n"
     ]
    }
   ],
   "source": [
    "X = nd.random.uniform(shape=(1, 1, 224, 224))\n",
    "net.initialize()\n",
    "for layer in net:\n",
    "    X = layer(X)\n",
    "    print(layer.name, 'output shape:\\t', X.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "3"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading /home/ubuntu/.mxnet/datasets/fashion-mnist/train-images-idx3-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/train-images-idx3-ubyte.gz...\n",
      "Downloading /home/ubuntu/.mxnet/datasets/fashion-mnist/train-labels-idx1-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/train-labels-idx1-ubyte.gz...\n",
      "Downloading /home/ubuntu/.mxnet/datasets/fashion-mnist/t10k-images-idx3-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/t10k-images-idx3-ubyte.gz...\n",
      "Downloading /home/ubuntu/.mxnet/datasets/fashion-mnist/t10k-labels-idx1-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/t10k-labels-idx1-ubyte.gz...\n"
     ]
    }
   ],
   "source": [
    "def load_data_fashion_mnist(batch_size, resize=None, root=os.path.join(\n",
    "        '~', '.mxnet', 'datasets', 'fashion-mnist')):\n",
    "    root = os.path.expanduser(root)  # Expand the user path '~'.\n",
    "    transformer = []\n",
    "    if resize:\n",
    "        transformer += [gdata.vision.transforms.Resize(resize)]\n",
    "    transformer += [gdata.vision.transforms.ToTensor()]\n",
    "    transformer = gdata.vision.transforms.Compose(transformer)\n",
    "    mnist_train = gdata.vision.FashionMNIST(root=root, train=True)\n",
    "    mnist_test = gdata.vision.FashionMNIST(root=root, train=False)\n",
    "    num_workers = 0 if sys.platform.startswith('win32') else 4\n",
    "    train_iter = gdata.DataLoader(\n",
    "        mnist_train.transform_first(transformer), batch_size, shuffle=True,\n",
    "        num_workers=num_workers)\n",
    "    test_iter = gdata.DataLoader(\n",
    "        mnist_test.transform_first(transformer), batch_size, shuffle=False,\n",
    "        num_workers=num_workers)\n",
    "    return train_iter, test_iter\n",
    "\n",
    "batch_size = 128\n",
    "train_iter, test_iter = load_data_fashion_mnist(batch_size, resize=224)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_accuracy(data_iter, net, ctx):\n",
    "    acc_sum, n = nd.array([0], ctx=ctx), 0\n",
    "    for X, y in data_iter:\n",
    "        # If ctx is the GPU, copy the data to the GPU.\n",
    "        X, y = X.as_in_context(ctx), y.as_in_context(ctx).astype('float32')\n",
    "        acc_sum += (net(X).argmax(axis=1) == y).sum()\n",
    "        n += y.size\n",
    "    return acc_sum.asscalar() / n\n",
    "\n",
    "def train(net, train_iter, test_iter, batch_size, trainer, ctx,\n",
    "              num_epochs):\n",
    "    print('training on', ctx)\n",
    "    loss = gloss.SoftmaxCrossEntropyLoss()\n",
    "    for epoch in range(num_epochs):\n",
    "        train_l_sum, train_acc_sum, n, start = 0.0, 0.0, 0, time.time()\n",
    "        for X, y in train_iter:\n",
    "            X, y = X.as_in_context(ctx), y.as_in_context(ctx)\n",
    "            with autograd.record():\n",
    "                y_hat = net(X)\n",
    "                l = loss(y_hat, y).sum()\n",
    "            l.backward()\n",
    "            trainer.step(batch_size)\n",
    "            y = y.astype('float32')\n",
    "            train_l_sum += l.asscalar()\n",
    "            train_acc_sum += (y_hat.argmax(axis=1) == y).sum().asscalar()\n",
    "            n += y.size\n",
    "        test_acc = evaluate_accuracy(test_iter, net, ctx)\n",
    "        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f, '\n",
    "              'time %.1f sec'\n",
    "              % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc,\n",
    "                 time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Compared to LeNet the main change is the smaller learning rate and much slower progress due to much larger images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "5"
    },
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training on gpu(0)\n",
      "epoch 1, loss 1.3052, train acc 0.509, test acc 0.745, time 18.9 sec\n",
      "epoch 2, loss 0.6477, train acc 0.756, test acc 0.815, time 22.5 sec\n",
      "epoch 3, loss 0.5331, train acc 0.803, test acc 0.831, time 16.6 sec\n",
      "epoch 4, loss 0.4651, train acc 0.829, test acc 0.855, time 16.6 sec\n",
      "epoch 5, loss 0.4276, train acc 0.844, test acc 0.867, time 16.6 sec\n"
     ]
    }
   ],
   "source": [
    "lr, num_epochs, ctx = 0.01, 5, d2l.try_gpu()\n",
    "net.initialize(force_reinit=True, ctx=ctx, init=init.Xavier())\n",
    "trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})\n",
    "train(net, train_iter, test_iter, batch_size, trainer, ctx, num_epochs)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
