transfer_learning_all_pretrained_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入必要的库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import gluon\n",
    "from mxnet import ndarray as nd\n",
    "from mxnet import autograd\n",
    "from mxnet.gluon import nn\n",
    "\n",
    "import h5py\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "ctx = mx.gpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "\n",
    "def accuracy(output, labels):\n",
    "    return nd.mean(nd.argmax(output, axis=1) == labels).asscalar()\n",
    "\n",
    "def evaluate(net, data_iter):\n",
    "    loss, acc, n = 0., 0., 0.\n",
    "    steps = len(data_iter)\n",
    "    for data, label in data_iter:\n",
    "        data, label = data.as_in_context(ctx), label.as_in_context(ctx)\n",
    "        output = net(data)\n",
    "        acc += accuracy(output, label)\n",
    "        loss += nd.mean(softmax_cross_entropy(output, label)).asscalar()\n",
    "    return loss/steps, acc/steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 写了一些函数\n",
    "\n",
    "* load_data 载入之前导出的特征\n",
    "* build_model 构建一个模型\n",
    "* train_model 输入预训练模型的名字，进行迁移学习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(model_name, batch_size=128, train_size=0.8):\n",
    "    features = nd.load('features_train_%s.nd' % model_name)[0]\n",
    "    labels = nd.load('labels.nd')[0]\n",
    "\n",
    "    n_train = int(features.shape[0]*train_size)\n",
    "\n",
    "    X_train = features[:n_train]\n",
    "    y_train = labels[:n_train]\n",
    "\n",
    "    X_val = features[n_train:]\n",
    "    y_val = labels[n_train:]\n",
    "\n",
    "    dataset_train = gluon.data.ArrayDataset(X_train, y_train)\n",
    "    dataset_val = gluon.data.ArrayDataset(X_val, y_val)\n",
    "\n",
    "    data_iter_train = gluon.data.DataLoader(dataset_train, batch_size)\n",
    "    data_iter_val = gluon.data.DataLoader(dataset_val, batch_size)\n",
    "    \n",
    "    return data_iter_train, data_iter_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    net = nn.Sequential()\n",
    "    with net.name_scope():\n",
    "        net.add(nn.Dense(256, activation='relu'))\n",
    "        net.add(nn.Dropout(0.5))\n",
    "        net.add(nn.Dense(120))\n",
    "\n",
    "    net.initialize(ctx=ctx)\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_model(model_name):\n",
    "    epochs = 50\n",
    "    batch_size = 128\n",
    "    \n",
    "    data_iter_train, data_iter_val = load_data(model_name, batch_size)\n",
    "    net = build_model()\n",
    "    \n",
    "    trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': 1e-4, 'wd': 1e-5})\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        train_loss = 0.\n",
    "        train_acc = 0.\n",
    "        steps = len(data_iter_train)\n",
    "        for data, label in data_iter_train:\n",
    "            data, label = data.as_in_context(ctx), label.as_in_context(ctx)\n",
    "\n",
    "            with autograd.record():\n",
    "                output = net(data)\n",
    "                loss = softmax_cross_entropy(output, label)\n",
    "\n",
    "            loss.backward()\n",
    "            trainer.step(batch_size)\n",
    "\n",
    "            train_loss += nd.mean(loss).asscalar()\n",
    "            train_acc += accuracy(output, label)\n",
    "\n",
    "        val_loss, val_acc = evaluate(net, data_iter_val)\n",
    "    \n",
    "    print(\"Epoch %d. loss: %.4f, acc: %.2f%%, val_loss %.4f, val_acc %.2f%% Model: %s\" % (\n",
    "        epoch+1, train_loss/steps, train_acc/steps*100, val_loss, val_acc*100, model_name))\n",
    "    \n",
    "    return val_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对所有预训练模型进行训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50. loss: 0.3581, acc: 91.84%, val_loss 1.7703, val_acc 53.16% Model: alexnet\n",
      "Epoch 50. loss: 0.5784, acc: 82.65%, val_loss 0.5917, val_acc 81.61% Model: densenet121\n",
      "Epoch 50. loss: 0.3868, acc: 88.30%, val_loss 0.4181, val_acc 86.60% Model: densenet161\n",
      "Epoch 50. loss: 0.4281, acc: 87.34%, val_loss 0.5125, val_acc 84.01% Model: densenet169\n",
      "Epoch 50. loss: 0.4015, acc: 88.35%, val_loss 0.4534, val_acc 85.09% Model: densenet201\n",
      "Epoch 50. loss: 0.2417, acc: 92.57%, val_loss 0.2961, val_acc 89.73% Model: inceptionv3\n",
      "Epoch 50. loss: 0.3471, acc: 89.42%, val_loss 0.4104, val_acc 87.29% Model: resnet101_v1\n",
      "Epoch 50. loss: 0.3468, acc: 89.39%, val_loss 0.3994, val_acc 87.39% Model: resnet152_v1\n",
      "Epoch 50. loss: 0.8433, acc: 74.35%, val_loss 0.8325, val_acc 74.19% Model: resnet18_v1\n",
      "Epoch 50. loss: 0.7811, acc: 76.53%, val_loss 0.7617, val_acc 76.53% Model: resnet18_v2\n",
      "Epoch 50. loss: 0.5969, acc: 82.09%, val_loss 0.5914, val_acc 82.10% Model: resnet34_v1\n",
      "Epoch 50. loss: 0.5438, acc: 83.38%, val_loss 0.5367, val_acc 83.22% Model: resnet34_v2\n",
      "Epoch 50. loss: 0.4418, acc: 86.28%, val_loss 0.4962, val_acc 83.86% Model: resnet50_v1\n",
      "Epoch 50. loss: 0.4827, acc: 84.71%, val_loss 0.4844, val_acc 84.79% Model: resnet50_v2\n",
      "Epoch 50. loss: 1.2832, acc: 62.63%, val_loss 1.6218, val_acc 54.19% Model: squeezenet1.0\n",
      "Epoch 50. loss: 1.2638, acc: 63.33%, val_loss 1.6067, val_acc 55.31% Model: squeezenet1.1\n",
      "Epoch 50. loss: 0.2596, acc: 93.47%, val_loss 0.7900, val_acc 75.60% Model: vgg11\n",
      "Epoch 50. loss: 0.3545, acc: 89.74%, val_loss 0.7084, val_acc 78.44% Model: vgg11_bn\n",
      "Epoch 50. loss: 0.2330, acc: 94.12%, val_loss 0.7565, val_acc 76.68% Model: vgg13\n",
      "Epoch 50. loss: 0.3428, acc: 90.39%, val_loss 0.7025, val_acc 77.36% Model: vgg13_bn\n",
      "Epoch 50. loss: 0.1944, acc: 95.13%, val_loss 0.6693, val_acc 79.81% Model: vgg16\n",
      "Epoch 50. loss: 0.2814, acc: 91.88%, val_loss 0.5865, val_acc 81.42% Model: vgg16_bn\n",
      "Epoch 50. loss: 0.1981, acc: 94.98%, val_loss 0.6198, val_acc 80.34% Model: vgg19\n",
      "Epoch 50. loss: 0.2630, acc: 92.49%, val_loss 0.5573, val_acc 81.91% Model: vgg19_bn\n"
     ]
    }
   ],
   "source": [
    "from mxnet.gluon.model_zoo.model_store import _model_sha1\n",
    "\n",
    "losses = []\n",
    "\n",
    "for model_name in sorted(_model_sha1.keys()):\n",
    "    val_loss = train_model(model_name)\n",
    "    losses.append((model_name, val_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对 val_loss 进行排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>inceptionv3</td>\n",
       "      <td>0.296050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>resnet152_v1</td>\n",
       "      <td>0.399360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>resnet101_v1</td>\n",
       "      <td>0.410383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>densenet161</td>\n",
       "      <td>0.418101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>densenet201</td>\n",
       "      <td>0.453403</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          model  val_loss\n",
       "5   inceptionv3  0.296050\n",
       "7  resnet152_v1  0.399360\n",
       "6  resnet101_v1  0.410383\n",
       "2   densenet161  0.418101\n",
       "4   densenet201  0.453403"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(losses, columns=['model', 'val_loss'])\n",
    "df = df.sort_values('val_loss')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('models.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('models.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inceptionv3 | 0.296050225385\n",
      "resnet152_v1 | 0.399359531701\n",
      "resnet101_v1 | 0.410383010283\n",
      "densenet161 | 0.418100789189\n",
      "densenet201 | 0.453403010964\n",
      "resnet50_v2 | 0.484435886145\n",
      "resnet50_v1 | 0.496179759502\n",
      "densenet169 | 0.512498702854\n",
      "resnet34_v2 | 0.536734519526\n",
      "vgg19_bn | 0.557294445112\n",
      "vgg16_bn | 0.586511127651\n",
      "resnet34_v1 | 0.591432901099\n",
      "densenet121 | 0.591716498137\n",
      "vgg19 | 0.619780953974\n",
      "vgg16 | 0.669267293066\n",
      "vgg13_bn | 0.702507363632\n",
      "vgg11_bn | 0.708396691829\n",
      "vgg13 | 0.756541173905\n",
      "resnet18_v2 | 0.761708110571\n",
      "vgg11 | 0.789955694228\n",
      "resnet18_v1 | 0.832537706941\n",
      "squeezenet1.1 | 1.6066500321\n",
      "squeezenet1.0 | 1.62178872526\n",
      "alexnet | 1.77026221156\n"
     ]
    }
   ],
   "source": [
    "for i, (model_name, val_loss) in df.iterrows():\n",
    "    print '%s | %s' % (model_name, val_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用效果最好的几个模型的特征进行训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_models_data(model_names, batch_size=128, train_size=0.8):\n",
    "    features = [nd.load('features_train_%s.nd' % model_name)[0] for model_name in model_names]\n",
    "    features = nd.concat(*features, dim=1)\n",
    "    labels = nd.load('labels.nd')[0]\n",
    "    \n",
    "    n_train = int(features.shape[0]*train_size)\n",
    "    \n",
    "    X_train = features[:n_train]\n",
    "    y_train = labels[:n_train]\n",
    "\n",
    "    X_val = features[n_train:]\n",
    "    y_val = labels[n_train:]\n",
    "\n",
    "    dataset_train = gluon.data.ArrayDataset(X_train, y_train)\n",
    "    dataset_val = gluon.data.ArrayDataset(X_val, y_val)\n",
    "\n",
    "    data_iter_train = gluon.data.DataLoader(dataset_train, batch_size, shuffle=True)\n",
    "    data_iter_val = gluon.data.DataLoader(dataset_val, batch_size)\n",
    "    \n",
    "    return data_iter_train, data_iter_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>inceptionv3</td>\n",
       "      <td>0.296050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>resnet152_v1</td>\n",
       "      <td>0.399360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>resnet101_v1</td>\n",
       "      <td>0.410383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>densenet161</td>\n",
       "      <td>0.418101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>densenet201</td>\n",
       "      <td>0.453403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>resnet50_v2</td>\n",
       "      <td>0.484436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>resnet50_v1</td>\n",
       "      <td>0.496180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>densenet169</td>\n",
       "      <td>0.512499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>resnet34_v2</td>\n",
       "      <td>0.536735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>vgg19_bn</td>\n",
       "      <td>0.557294</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          model  val_loss\n",
       "0   inceptionv3  0.296050\n",
       "1  resnet152_v1  0.399360\n",
       "2  resnet101_v1  0.410383\n",
       "3   densenet161  0.418101\n",
       "4   densenet201  0.453403\n",
       "5   resnet50_v2  0.484436\n",
       "6   resnet50_v1  0.496180\n",
       "7   densenet169  0.512499\n",
       "8   resnet34_v2  0.536735\n",
       "9      vgg19_bn  0.557294"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1. loss: 4.5554, acc: 6.17%, val_loss 3.9349, val_acc 35.16%\n",
      "Epoch 2. loss: 3.4425, acc: 28.50%, val_loss 2.6004, val_acc 63.87%\n",
      "Epoch 3. loss: 2.3478, acc: 51.45%, val_loss 1.5700, val_acc 79.51%\n",
      "Epoch 4. loss: 1.6228, acc: 65.17%, val_loss 1.0354, val_acc 84.90%\n",
      "Epoch 5. loss: 1.2263, acc: 72.43%, val_loss 0.7550, val_acc 87.34%\n",
      "Epoch 6. loss: 0.9907, acc: 76.36%, val_loss 0.6167, val_acc 88.07%\n",
      "Epoch 7. loss: 0.8461, acc: 79.29%, val_loss 0.5285, val_acc 88.56%\n",
      "Epoch 8. loss: 0.7600, acc: 80.44%, val_loss 0.4769, val_acc 89.68%\n",
      "Epoch 9. loss: 0.6722, acc: 82.62%, val_loss 0.4397, val_acc 89.34%\n",
      "Epoch 10. loss: 0.6256, acc: 83.62%, val_loss 0.4127, val_acc 89.49%\n",
      "Epoch 11. loss: 0.5917, acc: 84.15%, val_loss 0.3908, val_acc 90.12%\n",
      "Epoch 12. loss: 0.5380, acc: 85.39%, val_loss 0.3712, val_acc 89.83%\n",
      "Epoch 13. loss: 0.5155, acc: 85.85%, val_loss 0.3572, val_acc 90.37%\n",
      "Epoch 14. loss: 0.4787, acc: 86.82%, val_loss 0.3493, val_acc 90.12%\n",
      "Epoch 15. loss: 0.4646, acc: 87.07%, val_loss 0.3408, val_acc 89.97%\n",
      "Epoch 16. loss: 0.4414, acc: 87.71%, val_loss 0.3290, val_acc 90.27%\n",
      "Epoch 17. loss: 0.4274, acc: 87.88%, val_loss 0.3227, val_acc 90.41%\n",
      "Epoch 18. loss: 0.4067, acc: 88.78%, val_loss 0.3169, val_acc 90.41%\n",
      "Epoch 19. loss: 0.4061, acc: 88.46%, val_loss 0.3134, val_acc 90.51%\n",
      "Epoch 20. loss: 0.3785, acc: 89.15%, val_loss 0.3124, val_acc 90.12%\n",
      "Epoch 21. loss: 0.3607, acc: 89.84%, val_loss 0.3064, val_acc 90.22%\n",
      "Epoch 22. loss: 0.3590, acc: 89.71%, val_loss 0.3030, val_acc 90.32%\n",
      "Epoch 23. loss: 0.3565, acc: 89.64%, val_loss 0.3000, val_acc 90.22%\n",
      "Epoch 24. loss: 0.3397, acc: 89.89%, val_loss 0.2964, val_acc 90.41%\n",
      "Epoch 25. loss: 0.3304, acc: 90.53%, val_loss 0.2940, val_acc 90.66%\n",
      "Epoch 26. loss: 0.3278, acc: 90.52%, val_loss 0.2903, val_acc 90.51%\n",
      "Epoch 27. loss: 0.3179, acc: 90.64%, val_loss 0.2884, val_acc 90.75%\n",
      "Epoch 28. loss: 0.3085, acc: 91.17%, val_loss 0.2907, val_acc 90.66%\n",
      "Epoch 29. loss: 0.3074, acc: 91.05%, val_loss 0.2869, val_acc 90.51%\n",
      "Epoch 30. loss: 0.2879, acc: 91.76%, val_loss 0.2852, val_acc 90.61%\n",
      "Epoch 31. loss: 0.2944, acc: 91.50%, val_loss 0.2847, val_acc 90.61%\n",
      "Epoch 32. loss: 0.2853, acc: 91.64%, val_loss 0.2806, val_acc 90.61%\n",
      "Epoch 33. loss: 0.2690, acc: 92.29%, val_loss 0.2830, val_acc 90.41%\n",
      "Epoch 34. loss: 0.2694, acc: 92.08%, val_loss 0.2799, val_acc 90.85%\n",
      "Epoch 35. loss: 0.2671, acc: 92.44%, val_loss 0.2779, val_acc 90.51%\n",
      "Epoch 36. loss: 0.2615, acc: 92.13%, val_loss 0.2784, val_acc 90.61%\n",
      "Epoch 37. loss: 0.2565, acc: 92.39%, val_loss 0.2761, val_acc 90.71%\n",
      "Epoch 38. loss: 0.2537, acc: 92.58%, val_loss 0.2759, val_acc 90.36%\n",
      "Epoch 39. loss: 0.2511, acc: 92.51%, val_loss 0.2773, val_acc 90.66%\n",
      "Epoch 40. loss: 0.2402, acc: 92.97%, val_loss 0.2757, val_acc 90.71%\n",
      "Epoch 41. loss: 0.2373, acc: 93.17%, val_loss 0.2754, val_acc 90.95%\n",
      "Epoch 42. loss: 0.2306, acc: 92.94%, val_loss 0.2740, val_acc 90.66%\n",
      "Epoch 43. loss: 0.2363, acc: 93.13%, val_loss 0.2769, val_acc 90.71%\n",
      "Epoch 44. loss: 0.2350, acc: 92.67%, val_loss 0.2742, val_acc 90.56%\n",
      "Epoch 45. loss: 0.2233, acc: 93.42%, val_loss 0.2722, val_acc 90.81%\n",
      "Epoch 46. loss: 0.2211, acc: 93.44%, val_loss 0.2741, val_acc 90.51%\n",
      "Epoch 47. loss: 0.2175, acc: 93.88%, val_loss 0.2716, val_acc 90.61%\n",
      "Epoch 48. loss: 0.2167, acc: 93.38%, val_loss 0.2716, val_acc 90.76%\n",
      "Epoch 49. loss: 0.2162, acc: 93.68%, val_loss 0.2725, val_acc 90.71%\n",
      "Epoch 50. loss: 0.2094, acc: 93.73%, val_loss 0.2725, val_acc 91.00%\n",
      "Epoch 51. loss: 0.2079, acc: 93.89%, val_loss 0.2716, val_acc 90.46%\n",
      "Epoch 52. loss: 0.2014, acc: 94.42%, val_loss 0.2721, val_acc 90.61%\n",
      "Epoch 53. loss: 0.2009, acc: 93.99%, val_loss 0.2702, val_acc 90.86%\n",
      "Epoch 54. loss: 0.2036, acc: 93.93%, val_loss 0.2714, val_acc 90.61%\n",
      "Epoch 55. loss: 0.2004, acc: 94.31%, val_loss 0.2710, val_acc 90.46%\n",
      "Epoch 56. loss: 0.1981, acc: 94.35%, val_loss 0.2713, val_acc 90.51%\n",
      "Epoch 57. loss: 0.1870, acc: 94.79%, val_loss 0.2682, val_acc 90.46%\n",
      "Epoch 58. loss: 0.1933, acc: 94.44%, val_loss 0.2687, val_acc 90.76%\n",
      "Epoch 59. loss: 0.1864, acc: 94.65%, val_loss 0.2687, val_acc 90.71%\n",
      "Epoch 60. loss: 0.1805, acc: 94.78%, val_loss 0.2690, val_acc 90.56%\n",
      "Epoch 61. loss: 0.1848, acc: 94.63%, val_loss 0.2674, val_acc 90.71%\n",
      "Epoch 62. loss: 0.1794, acc: 94.88%, val_loss 0.2701, val_acc 90.81%\n",
      "Epoch 63. loss: 0.1803, acc: 94.66%, val_loss 0.2689, val_acc 90.61%\n",
      "Epoch 64. loss: 0.1795, acc: 94.96%, val_loss 0.2684, val_acc 90.61%\n",
      "Epoch 65. loss: 0.1744, acc: 94.89%, val_loss 0.2682, val_acc 90.76%\n",
      "Epoch 66. loss: 0.1724, acc: 95.08%, val_loss 0.2694, val_acc 90.56%\n",
      "Epoch 67. loss: 0.1735, acc: 94.81%, val_loss 0.2672, val_acc 90.90%\n",
      "Epoch 68. loss: 0.1636, acc: 95.35%, val_loss 0.2668, val_acc 90.90%\n",
      "Epoch 69. loss: 0.1675, acc: 95.08%, val_loss 0.2662, val_acc 90.85%\n",
      "Epoch 70. loss: 0.1715, acc: 94.96%, val_loss 0.2676, val_acc 90.76%\n",
      "Epoch 71. loss: 0.1681, acc: 95.19%, val_loss 0.2666, val_acc 90.66%\n",
      "Epoch 72. loss: 0.1592, acc: 95.68%, val_loss 0.2675, val_acc 90.80%\n",
      "Epoch 73. loss: 0.1648, acc: 95.11%, val_loss 0.2676, val_acc 90.71%\n",
      "Epoch 74. loss: 0.1638, acc: 95.44%, val_loss 0.2675, val_acc 90.66%\n",
      "Epoch 75. loss: 0.1563, acc: 95.44%, val_loss 0.2676, val_acc 90.66%\n",
      "Epoch 76. loss: 0.1566, acc: 95.66%, val_loss 0.2674, val_acc 90.71%\n",
      "Epoch 77. loss: 0.1532, acc: 95.61%, val_loss 0.2673, val_acc 90.71%\n",
      "Epoch 78. loss: 0.1586, acc: 95.54%, val_loss 0.2665, val_acc 90.85%\n",
      "Epoch 79. loss: 0.1489, acc: 96.10%, val_loss 0.2660, val_acc 90.56%\n",
      "Epoch 80. loss: 0.1536, acc: 95.54%, val_loss 0.2673, val_acc 90.81%\n",
      "Epoch 81. loss: 0.1515, acc: 95.67%, val_loss 0.2679, val_acc 90.76%\n",
      "Epoch 82. loss: 0.1510, acc: 95.66%, val_loss 0.2674, val_acc 90.81%\n",
      "Epoch 83. loss: 0.1483, acc: 95.66%, val_loss 0.2671, val_acc 90.61%\n",
      "Epoch 84. loss: 0.1467, acc: 95.73%, val_loss 0.2687, val_acc 90.56%\n",
      "Epoch 85. loss: 0.1493, acc: 95.76%, val_loss 0.2679, val_acc 90.61%\n",
      "Epoch 86. loss: 0.1482, acc: 95.78%, val_loss 0.2675, val_acc 90.56%\n",
      "Epoch 87. loss: 0.1455, acc: 95.84%, val_loss 0.2680, val_acc 90.76%\n",
      "Epoch 88. loss: 0.1465, acc: 95.78%, val_loss 0.2673, val_acc 90.80%\n",
      "Epoch 89. loss: 0.1457, acc: 95.83%, val_loss 0.2677, val_acc 90.90%\n",
      "Epoch 90. loss: 0.1373, acc: 96.17%, val_loss 0.2669, val_acc 90.80%\n",
      "Epoch 91. loss: 0.1334, acc: 96.26%, val_loss 0.2679, val_acc 90.61%\n",
      "Epoch 92. loss: 0.1382, acc: 96.05%, val_loss 0.2668, val_acc 90.80%\n",
      "Epoch 93. loss: 0.1359, acc: 96.18%, val_loss 0.2681, val_acc 90.61%\n",
      "Epoch 94. loss: 0.1385, acc: 96.42%, val_loss 0.2687, val_acc 90.61%\n",
      "Epoch 95. loss: 0.1370, acc: 96.19%, val_loss 0.2670, val_acc 90.71%\n",
      "Epoch 96. loss: 0.1379, acc: 96.19%, val_loss 0.2664, val_acc 90.90%\n",
      "Epoch 97. loss: 0.1329, acc: 96.34%, val_loss 0.2664, val_acc 90.90%\n",
      "Epoch 98. loss: 0.1397, acc: 96.19%, val_loss 0.2677, val_acc 90.80%\n",
      "Epoch 99. loss: 0.1351, acc: 96.04%, val_loss 0.2654, val_acc 90.80%\n",
      "Epoch 100. loss: 0.1320, acc: 96.18%, val_loss 0.2662, val_acc 90.66%\n"
     ]
    }
   ],
   "source": [
    "net = build_model()\n",
    "\n",
    "model_names = ['inceptionv3', 'resnet152_v1']\n",
    "data_iter_train, data_iter_val = load_models_data(model_names, batch_size=batch_size)\n",
    "\n",
    "epochs = 100\n",
    "batch_size = 128\n",
    "lr_sch = mx.lr_scheduler.FactorScheduler(step=400, factor=0.9)\n",
    "trainer = gluon.Trainer(net.collect_params(), 'adam', \n",
    "                        {'learning_rate': 1e-4, 'wd': 1e-5, 'lr_scheduler': lr_sch})\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    train_loss = 0.\n",
    "    train_acc = 0.\n",
    "    steps = len(data_iter_train)\n",
    "    for data, label in data_iter_train:\n",
    "        data, label = data.as_in_context(ctx), label.as_in_context(ctx)\n",
    "\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            loss = softmax_cross_entropy(output, label)\n",
    "\n",
    "        loss.backward()\n",
    "        trainer.step(batch_size)\n",
    "\n",
    "        train_loss += nd.mean(loss).asscalar()\n",
    "        train_acc += accuracy(output, label)\n",
    "\n",
    "    val_loss, val_acc = evaluate(net, data_iter_val)\n",
    "\n",
    "    print(\"Epoch %d. loss: %.4f, acc: %.2f%%, val_loss %.4f, val_acc %.2f%%\" % (\n",
    "        epoch+1, train_loss/steps, train_acc/steps*100, val_loss, val_acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 载入测试集特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_test = [nd.load('features_test_%s.nd' % model_name)[0] for model_name in model_names]\n",
    "features_test = nd.concat(*features_test, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 利用模型进行预测并输出到 pred.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output = nd.softmax(net(features_test.as_in_context(ctx))).asnumpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_pred = pd.read_csv('sample_submission.csv')\n",
    "\n",
    "for i, c in enumerate(df_pred.columns[1:]):\n",
    "    df_pred[c] = output[:,i]\n",
    "\n",
    "df_pred.to_csv('pred.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 与上次提交结果进行比对，防止错位浪费机会"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(61, 61),\n",
       " (94, 94),\n",
       " (40, 40),\n",
       " (88, 88),\n",
       " (70, 70),\n",
       " (76, 76),\n",
       " (6, 6),\n",
       " (94, 94),\n",
       " (59, 59),\n",
       " (107, 107)]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zip(np.argmax(pd.read_csv('pred_0.28.csv').values[:,1:], axis=-1), np.argmax(df_pred.values[:,1:], axis=-1))[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}