CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import max_features, maxlen, embed_size_fastText, embed_size_glove, embed_size_glove_twitter\n",
    "from utils import batch_size, epochs, run_5fold, write_prediction, eval_results\n",
    "from model import get_CNN_model\n",
    "import pickle\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "fastText = pickle.load(open(\"input/fastText.300d.pickle\", \"rb\"))\n",
    "glove = pickle.load(open(\"input/glove.300d.pickle\", \"rb\"))\n",
    "glove_twitter = pickle.load(open(\"input/glove.twitter.200d.pickle\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = pickle.load(open(\"input/x_train.pickle\", \"rb\"))\n",
    "y_train = pickle.load(open(\"input/y_train.pickle\", \"rb\"))\n",
    "x_test = pickle.load(open(\"input/x_test.pickle\", \"rb\"))\n",
    "submission = pd.read_csv(\"input/sample_submission.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check different embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"dropout\": 0.2,\n",
    "    \"num_filter\": 50,\n",
    "    \"kernel_size\": 3,\n",
    "    \"reg\": 0.0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = [(fastText, embed_size_fastText, \"fastText\"), (glove, embed_size_glove, \"glove\"), (glove_twitter, embed_size_glove_twitter, \"glove_twitter\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "********************fastText********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0697 - acc: 0.9760 - auc_roc: 0.8875 - val_loss: 0.0483 - val_acc: 0.9817 - val_auc_roc: 0.9624\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0424 - acc: 0.9841 - auc_roc: 0.9720 - val_loss: 0.0454 - val_acc: 0.9825 - val_auc_roc: 0.9771\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0350 - acc: 0.9865 - auc_roc: 0.9803 - val_loss: 0.0463 - val_acc: 0.9827 - val_auc_roc: 0.9824\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04627582911981529\n",
      "val acc: 0.9827301460337381\n",
      "val auc roc: 0.9823546823860615\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0645 - acc: 0.9784 - auc_roc: 0.9056 - val_loss: 0.0478 - val_acc: 0.9819 - val_auc_roc: 0.9683\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0416 - acc: 0.9842 - auc_roc: 0.9758 - val_loss: 0.0455 - val_acc: 0.9824 - val_auc_roc: 0.9799\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0344 - acc: 0.9866 - auc_roc: 0.9825 - val_loss: 0.0456 - val_acc: 0.9829 - val_auc_roc: 0.9843\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04557888745769467\n",
      "val acc: 0.9829385014429587\n",
      "val auc roc: 0.9842705616692803\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0662 - acc: 0.9777 - auc_roc: 0.9007 - val_loss: 0.0451 - val_acc: 0.9831 - val_auc_roc: 0.9645\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0413 - acc: 0.9841 - auc_roc: 0.9736 - val_loss: 0.0434 - val_acc: 0.9835 - val_auc_roc: 0.9784\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0347 - acc: 0.9865 - auc_roc: 0.9813 - val_loss: 0.0444 - val_acc: 0.9833 - val_auc_roc: 0.9832\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.044392053243091485\n",
      "val acc: 0.9832779559872761\n",
      "val auc roc: 0.9832112465562501\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0710 - acc: 0.9762 - auc_roc: 0.8850 - val_loss: 0.0493 - val_acc: 0.9818 - val_auc_roc: 0.9604\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0435 - acc: 0.9836 - auc_roc: 0.9705 - val_loss: 0.0458 - val_acc: 0.9825 - val_auc_roc: 0.9759\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0362 - acc: 0.9860 - auc_roc: 0.9794 - val_loss: 0.0450 - val_acc: 0.9828 - val_auc_roc: 0.9816\n",
      "Epoch 4/50\n",
      " - 29s - loss: 0.0302 - acc: 0.9882 - auc_roc: 0.9834 - val_loss: 0.0483 - val_acc: 0.9825 - val_auc_roc: 0.9846\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04826375723264268\n",
      "val acc: 0.9824998254217794\n",
      "val auc roc: 0.9845858612230579\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0717 - acc: 0.9757 - auc_roc: 0.8834 - val_loss: 0.0484 - val_acc: 0.9818 - val_auc_roc: 0.9637\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0425 - acc: 0.9839 - auc_roc: 0.9733 - val_loss: 0.0457 - val_acc: 0.9826 - val_auc_roc: 0.9783\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0354 - acc: 0.9863 - auc_roc: 0.9813 - val_loss: 0.0460 - val_acc: 0.9827 - val_auc_roc: 0.9832\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.045988057031662546\n",
      "val acc: 0.9827400497074764\n",
      "val auc roc: 0.9832182278338475\n",
      "\n",
      "\n",
      "********************glove********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0688 - acc: 0.9754 - auc_roc: 0.8955 - val_loss: 0.0468 - val_acc: 0.9824 - val_auc_roc: 0.9672\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0426 - acc: 0.9839 - auc_roc: 0.9751 - val_loss: 0.0461 - val_acc: 0.9825 - val_auc_roc: 0.9791\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0365 - acc: 0.9860 - auc_roc: 0.9817 - val_loss: 0.0454 - val_acc: 0.9826 - val_auc_roc: 0.9834\n",
      "Epoch 4/50\n",
      " - 29s - loss: 0.0309 - acc: 0.9881 - auc_roc: 0.9848 - val_loss: 0.0463 - val_acc: 0.9826 - val_auc_roc: 0.9858\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.0463346261544538\n",
      "val acc: 0.982568261587151\n",
      "val auc roc: 0.9858225075368987\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0641 - acc: 0.9783 - auc_roc: 0.9093 - val_loss: 0.0464 - val_acc: 0.9824 - val_auc_roc: 0.9696\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0419 - acc: 0.9841 - auc_roc: 0.9761 - val_loss: 0.0447 - val_acc: 0.9829 - val_auc_roc: 0.9799\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0358 - acc: 0.9862 - auc_roc: 0.9821 - val_loss: 0.0455 - val_acc: 0.9824 - val_auc_roc: 0.9838\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.045484460354273414\n",
      "val acc: 0.9824162662503653\n",
      "val auc roc: 0.9837796789979609\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0645 - acc: 0.9769 - auc_roc: 0.9112 - val_loss: 0.0448 - val_acc: 0.9831 - val_auc_roc: 0.9719\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0421 - acc: 0.9840 - auc_roc: 0.9781 - val_loss: 0.0442 - val_acc: 0.9834 - val_auc_roc: 0.9812\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0358 - acc: 0.9862 - auc_roc: 0.9833 - val_loss: 0.0444 - val_acc: 0.9834 - val_auc_roc: 0.9846\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04437776614757247\n",
      "val acc: 0.9834294013079974\n",
      "val auc roc: 0.9846269969445031\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0595 - acc: 0.9796 - auc_roc: 0.9253 - val_loss: 0.0461 - val_acc: 0.9827 - val_auc_roc: 0.9735\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0413 - acc: 0.9843 - auc_roc: 0.9790 - val_loss: 0.0442 - val_acc: 0.9834 - val_auc_roc: 0.9818\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0353 - acc: 0.9863 - auc_roc: 0.9838 - val_loss: 0.0460 - val_acc: 0.9829 - val_auc_roc: 0.9852\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.045979295908086186\n",
      "val acc: 0.9828862750889582\n",
      "val auc roc: 0.9851651507720258\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 29s - loss: 0.0606 - acc: 0.9784 - auc_roc: 0.9240 - val_loss: 0.0458 - val_acc: 0.9825 - val_auc_roc: 0.9734\n",
      "Epoch 2/50\n",
      " - 29s - loss: 0.0414 - acc: 0.9841 - auc_roc: 0.9790 - val_loss: 0.0441 - val_acc: 0.9833 - val_auc_roc: 0.9818\n",
      "Epoch 3/50\n",
      " - 29s - loss: 0.0352 - acc: 0.9863 - auc_roc: 0.9838 - val_loss: 0.0458 - val_acc: 0.9827 - val_auc_roc: 0.9851\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04578627270055219\n",
      "val acc: 0.9826826015241599\n",
      "val auc roc: 0.9851050803690781\n",
      "\n",
      "\n",
      "********************glove_twitter********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0656 - acc: 0.9778 - auc_roc: 0.9089 - val_loss: 0.0471 - val_acc: 0.9822 - val_auc_roc: 0.9682\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0434 - acc: 0.9837 - auc_roc: 0.9755 - val_loss: 0.0449 - val_acc: 0.9828 - val_auc_roc: 0.9794\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0377 - acc: 0.9856 - auc_roc: 0.9817 - val_loss: 0.0449 - val_acc: 0.9831 - val_auc_roc: 0.9834\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04494266840692749\n",
      "val acc: 0.9831061458972373\n",
      "val auc roc: 0.983398223623631\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0695 - acc: 0.9756 - auc_roc: 0.9005 - val_loss: 0.0485 - val_acc: 0.9821 - val_auc_roc: 0.9665\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0437 - acc: 0.9834 - auc_roc: 0.9743 - val_loss: 0.0450 - val_acc: 0.9831 - val_auc_roc: 0.9786\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0382 - acc: 0.9852 - auc_roc: 0.9812 - val_loss: 0.0451 - val_acc: 0.9830 - val_auc_roc: 0.9829\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.045066483510241305\n",
      "val acc: 0.9830325062265819\n",
      "val auc roc: 0.9828999201262468\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0660 - acc: 0.9774 - auc_roc: 0.9100 - val_loss: 0.0462 - val_acc: 0.9830 - val_auc_roc: 0.9695\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0438 - acc: 0.9835 - auc_roc: 0.9763 - val_loss: 0.0440 - val_acc: 0.9840 - val_auc_roc: 0.9796\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0382 - acc: 0.9853 - auc_roc: 0.9819 - val_loss: 0.0450 - val_acc: 0.9831 - val_auc_roc: 0.9834\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.044989339649880854\n",
      "val acc: 0.983105612591543\n",
      "val auc roc: 0.9834348726909083\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 22s - loss: 0.0630 - acc: 0.9789 - auc_roc: 0.9165 - val_loss: 0.0462 - val_acc: 0.9826 - val_auc_roc: 0.9693\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0429 - acc: 0.9838 - auc_roc: 0.9764 - val_loss: 0.0452 - val_acc: 0.9827 - val_auc_roc: 0.9800\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0377 - acc: 0.9855 - auc_roc: 0.9823 - val_loss: 0.0451 - val_acc: 0.9828 - val_auc_roc: 0.9838\n",
      "Epoch 4/50\n",
      " - 21s - loss: 0.0332 - acc: 0.9871 - auc_roc: 0.9851 - val_loss: 0.0469 - val_acc: 0.9821 - val_auc_roc: 0.9860\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.046862894996133164\n",
      "val acc: 0.9820715886590146\n",
      "val auc roc: 0.9860089914634531\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0643 - acc: 0.9780 - auc_roc: 0.9160 - val_loss: 0.0476 - val_acc: 0.9821 - val_auc_roc: 0.9707\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/50\n",
      " - 21s - loss: 0.0433 - acc: 0.9836 - auc_roc: 0.9771 - val_loss: 0.0443 - val_acc: 0.9832 - val_auc_roc: 0.9803\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0378 - acc: 0.9853 - auc_roc: 0.9825 - val_loss: 0.0446 - val_acc: 0.9833 - val_auc_roc: 0.9839\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04464892064548453\n",
      "val acc: 0.9832988464384813\n",
      "val auc roc: 0.9839194666445087\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for embed in embeddings:\n",
    "    print (\"\\n********************{}********************\".format(embed[2]))\n",
    "    params[\"embedding_matrix\"] = embed[0]\n",
    "    params[\"embed_size\"] = embed[1]\n",
    "    y_preds = run_5fold(x_train, y_train, x_test, \"loss\", get_CNN_model, **params)\n",
    "    predictions[embed[2]] = y_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: fastText\n",
      "loss: 0.04609971681698133 (+/- 0.0012580884288926478)\n",
      "acc: 0.9828372957186456 (+/- 0.0002604796014717037)\n",
      "roc_auc: 0.9835281159336994 (+/- 0.0008054109379638884)\n",
      "\n",
      "\n",
      "\n",
      "key: glove\n",
      "loss: 0.04559248425298761 (+/- 0.00066710734655799)\n",
      "acc: 0.9827965611517264 (+/- 0.0003516438407890546)\n",
      "roc_auc: 0.9848998829240934 (+/- 0.0006772049605117584)\n",
      "\n",
      "\n",
      "\n",
      "key: glove_twitter\n",
      "loss: 0.045302061441733474 (+/- 0.0007931266631046235)\n",
      "acc: 0.9829229399625717 (+/- 0.0004347512370955112)\n",
      "roc_auc: 0.9839322949097496 (+/- 0.0010873170500473907)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_results(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### result\n",
    "- fastText: \n",
    "\n",
    "loss: 0.04609971681698133 (+/- 0.0012580884288926478)\n",
    "\n",
    "acc: 0.9828372957186456 (+/- 0.0002604796014717037)\n",
    "\n",
    "roc_auc: 0.9835281159336994 (+/- 0.0008054109379638884)\n",
    "\n",
    "- glove: \n",
    "\n",
    "loss: 0.04559248425298761 (+/- 0.00066710734655799)\n",
    "\n",
    "acc: 0.9827965611517264 (+/- 0.0003516438407890546)\n",
    "\n",
    "roc_auc: 0.9848998829240934 (+/- 0.0006772049605117584)\n",
    "\n",
    "- glove.twitter: \n",
    "\n",
    "loss: 0.045302061441733474 (+/- 0.0007931266631046235)\n",
    "\n",
    "acc: 0.9829229399625717 (+/- 0.0004347512370955112)\n",
    "\n",
    "roc_auc: 0.9839322949097496 (+/- 0.0010873170500473907)\n",
    "\n",
    "=============================================\n",
    "\n",
    "- mini conclusion:\n",
    "\n",
    "multi channel: glove, glove.twitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_glove = predictions['glove']['pred']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_twitter = predictions['glove_twitter']['pred']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# single model for glove twitter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check num_filter\n",
    "(suggest values: 100~600: test 100, 300, 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = [100, 300, 500]  # for the result of 50, refer to the previous log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"dropout\": 0.2,\n",
    "    \"kernel_size\": 3,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove_twitter,\n",
    "    \"embed_size\": embed_size_glove_twitter\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_twitter = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "********************f = 100********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0629 - acc: 0.9781 - auc_roc: 0.9161 - val_loss: 0.0472 - val_acc: 0.9822 - val_auc_roc: 0.9715\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0422 - acc: 0.9839 - auc_roc: 0.9778 - val_loss: 0.0462 - val_acc: 0.9822 - val_auc_roc: 0.9810\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0363 - acc: 0.9859 - auc_roc: 0.9832 - val_loss: 0.0440 - val_acc: 0.9836 - val_auc_roc: 0.9847\n",
      "Epoch 4/50\n",
      " - 26s - loss: 0.0315 - acc: 0.9878 - auc_roc: 0.9859 - val_loss: 0.0458 - val_acc: 0.9829 - val_auc_roc: 0.9868\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04583091041125438\n",
      "val acc: 0.982871147235247\n",
      "val auc roc: 0.9867898948435758\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0634 - acc: 0.9776 - auc_roc: 0.9158 - val_loss: 0.0467 - val_acc: 0.9829 - val_auc_roc: 0.9719\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0423 - acc: 0.9840 - auc_roc: 0.9778 - val_loss: 0.0446 - val_acc: 0.9831 - val_auc_roc: 0.9810\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0366 - acc: 0.9858 - auc_roc: 0.9832 - val_loss: 0.0453 - val_acc: 0.9831 - val_auc_roc: 0.9846\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04526333039430491\n",
      "val acc: 0.9830586132477599\n",
      "val auc roc: 0.9846406965474777\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0637 - acc: 0.9782 - auc_roc: 0.9158 - val_loss: 0.0454 - val_acc: 0.9833 - val_auc_roc: 0.9705\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0421 - acc: 0.9839 - auc_roc: 0.9770 - val_loss: 0.0430 - val_acc: 0.9839 - val_auc_roc: 0.9806\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0365 - acc: 0.9858 - auc_roc: 0.9828 - val_loss: 0.0437 - val_acc: 0.9835 - val_auc_roc: 0.9844\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04371906050516563\n",
      "val acc: 0.983549516579184\n",
      "val auc roc: 0.9843765512074251\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0651 - acc: 0.9769 - auc_roc: 0.9134 - val_loss: 0.0462 - val_acc: 0.9828 - val_auc_roc: 0.9712\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0426 - acc: 0.9838 - auc_roc: 0.9776 - val_loss: 0.0435 - val_acc: 0.9834 - val_auc_roc: 0.9808\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0366 - acc: 0.9858 - auc_roc: 0.9830 - val_loss: 0.0441 - val_acc: 0.9833 - val_auc_roc: 0.9844\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04413795685459216\n",
      "val acc: 0.9832831757883085\n",
      "val auc roc: 0.9844388714924099\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0675 - acc: 0.9765 - auc_roc: 0.9093 - val_loss: 0.0470 - val_acc: 0.9826 - val_auc_roc: 0.9697\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0427 - acc: 0.9838 - auc_roc: 0.9765 - val_loss: 0.0456 - val_acc: 0.9826 - val_auc_roc: 0.9801\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0370 - acc: 0.9857 - auc_roc: 0.9824 - val_loss: 0.0449 - val_acc: 0.9832 - val_auc_roc: 0.9839\n",
      "Epoch 4/50\n",
      " - 26s - loss: 0.0321 - acc: 0.9876 - auc_roc: 0.9852 - val_loss: 0.0484 - val_acc: 0.9822 - val_auc_roc: 0.9861\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04835869517288703\n",
      "val acc: 0.982249148109245\n",
      "val auc roc: 0.9861109007822061\n",
      "\n",
      "\n",
      "********************f = 300********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 42s - loss: 0.0550 - acc: 0.9806 - auc_roc: 0.9466 - val_loss: 0.0448 - val_acc: 0.9829 - val_auc_roc: 0.9781\n",
      "Epoch 2/50\n",
      " - 42s - loss: 0.0401 - acc: 0.9847 - auc_roc: 0.9822 - val_loss: 0.0435 - val_acc: 0.9833 - val_auc_roc: 0.9843\n",
      "Epoch 3/50\n",
      " - 42s - loss: 0.0339 - acc: 0.9868 - auc_roc: 0.9859 - val_loss: 0.0463 - val_acc: 0.9824 - val_auc_roc: 0.9869\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.046338047915242334\n",
      "val acc: 0.9824429290731294\n",
      "val auc roc: 0.9869428679540803\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 42s - loss: 0.0555 - acc: 0.9808 - auc_roc: 0.9426 - val_loss: 0.0442 - val_acc: 0.9832 - val_auc_roc: 0.9775\n",
      "Epoch 2/50\n",
      " - 42s - loss: 0.0401 - acc: 0.9847 - auc_roc: 0.9816 - val_loss: 0.0435 - val_acc: 0.9836 - val_auc_roc: 0.9839\n",
      "Epoch 3/50\n",
      " - 42s - loss: 0.0340 - acc: 0.9868 - auc_roc: 0.9855 - val_loss: 0.0448 - val_acc: 0.9833 - val_auc_roc: 0.9865\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04478210943858486\n",
      "val acc: 0.983272735394354\n",
      "val auc roc: 0.9865274742453919\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 42s - loss: 0.0574 - acc: 0.9796 - auc_roc: 0.9367 - val_loss: 0.0483 - val_acc: 0.9820 - val_auc_roc: 0.9764\n",
      "Epoch 2/50\n",
      " - 42s - loss: 0.0408 - acc: 0.9844 - auc_roc: 0.9809 - val_loss: 0.0426 - val_acc: 0.9838 - val_auc_roc: 0.9833\n",
      "Epoch 3/50\n",
      " - 42s - loss: 0.0348 - acc: 0.9865 - auc_roc: 0.9851 - val_loss: 0.0433 - val_acc: 0.9840 - val_auc_roc: 0.9862\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04326977523962059\n",
      "val acc: 0.98395686472479\n",
      "val auc roc: 0.9861888506138035\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 42s - loss: 0.0593 - acc: 0.9786 - auc_roc: 0.9309 - val_loss: 0.0454 - val_acc: 0.9831 - val_auc_roc: 0.9751\n",
      "Epoch 2/50\n",
      " - 42s - loss: 0.0407 - acc: 0.9844 - auc_roc: 0.9803 - val_loss: 0.0427 - val_acc: 0.9838 - val_auc_roc: 0.9829\n",
      "Epoch 3/50\n",
      " - 42s - loss: 0.0347 - acc: 0.9865 - auc_roc: 0.9848 - val_loss: 0.0443 - val_acc: 0.9834 - val_auc_roc: 0.9860\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.0443478075936488\n",
      "val acc: 0.9834294035193122\n",
      "val auc roc: 0.9859645040798599\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 42s - loss: 0.0577 - acc: 0.9795 - auc_roc: 0.9320 - val_loss: 0.0454 - val_acc: 0.9828 - val_auc_roc: 0.9759\n",
      "Epoch 2/50\n",
      " - 42s - loss: 0.0404 - acc: 0.9845 - auc_roc: 0.9808 - val_loss: 0.0448 - val_acc: 0.9831 - val_auc_roc: 0.9830\n",
      "Epoch 3/50\n",
      " - 42s - loss: 0.0343 - acc: 0.9867 - auc_roc: 0.9847 - val_loss: 0.0446 - val_acc: 0.9831 - val_auc_roc: 0.9859\n",
      "Epoch 4/50\n",
      " - 42s - loss: 0.0289 - acc: 0.9886 - auc_roc: 0.9871 - val_loss: 0.0507 - val_acc: 0.9811 - val_auc_roc: 0.9879\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.050666683448442745\n",
      "val acc: 0.9811420064376785\n",
      "val auc roc: 0.9879295996438667\n",
      "\n",
      "\n",
      "********************f = 500********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 55s - loss: 0.0564 - acc: 0.9801 - auc_roc: 0.9365 - val_loss: 0.0454 - val_acc: 0.9827 - val_auc_roc: 0.9766\n",
      "Epoch 2/50\n",
      " - 54s - loss: 0.0403 - acc: 0.9846 - auc_roc: 0.9812 - val_loss: 0.0445 - val_acc: 0.9836 - val_auc_roc: 0.9832\n",
      "Epoch 3/50\n",
      " - 54s - loss: 0.0335 - acc: 0.9869 - auc_roc: 0.9849 - val_loss: 0.0466 - val_acc: 0.9823 - val_auc_roc: 0.9861\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.046612262606886924\n",
      "val acc: 0.9822758146910837\n",
      "val auc roc: 0.9861096453203521\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 55s - loss: 0.0561 - acc: 0.9801 - auc_roc: 0.9397 - val_loss: 0.0455 - val_acc: 0.9827 - val_auc_roc: 0.9767\n",
      "Epoch 2/50\n",
      " - 54s - loss: 0.0400 - acc: 0.9846 - auc_roc: 0.9811 - val_loss: 0.0443 - val_acc: 0.9834 - val_auc_roc: 0.9834\n",
      "Epoch 3/50\n",
      " - 54s - loss: 0.0338 - acc: 0.9869 - auc_roc: 0.9850 - val_loss: 0.0451 - val_acc: 0.9832 - val_auc_roc: 0.9862\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04510333814311014\n",
      "val acc: 0.9832413969590056\n",
      "val auc roc: 0.9861724167537397\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 55s - loss: 0.0559 - acc: 0.9803 - auc_roc: 0.9427 - val_loss: 0.0432 - val_acc: 0.9838 - val_auc_roc: 0.9769\n",
      "Epoch 2/50\n",
      " - 54s - loss: 0.0404 - acc: 0.9845 - auc_roc: 0.9814 - val_loss: 0.0464 - val_acc: 0.9825 - val_auc_roc: 0.9835\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.04643253982323912\n",
      "val acc: 0.9825468212207352\n",
      "val auc roc: 0.9834912314281702\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 55s - loss: 0.0561 - acc: 0.9801 - auc_roc: 0.9373 - val_loss: 0.0429 - val_acc: 0.9835 - val_auc_roc: 0.9767\n",
      "Epoch 2/50\n",
      " - 54s - loss: 0.0399 - acc: 0.9847 - auc_roc: 0.9814 - val_loss: 0.0430 - val_acc: 0.9837 - val_auc_roc: 0.9837\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.04301047994225084\n",
      "val acc: 0.9836748515094604\n",
      "val auc roc: 0.9837041823287398\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 55s - loss: 0.0562 - acc: 0.9801 - auc_roc: 0.9387 - val_loss: 0.0455 - val_acc: 0.9829 - val_auc_roc: 0.9765\n",
      "Epoch 2/50\n",
      " - 54s - loss: 0.0401 - acc: 0.9846 - auc_roc: 0.9812 - val_loss: 0.0437 - val_acc: 0.9830 - val_auc_roc: 0.9834\n",
      "Epoch 3/50\n",
      " - 54s - loss: 0.0337 - acc: 0.9867 - auc_roc: 0.9853 - val_loss: 0.0455 - val_acc: 0.9830 - val_auc_roc: 0.9863\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00003: early stopping\n",
      "val loss: 0.045533780417572986\n",
      "val acc: 0.9830325051694837\n",
      "val auc roc: 0.9863445628830131\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for f in filters:\n",
    "    print (\"\\n********************f = {}********************\".format(f))\n",
    "    params[\"num_filter\"] = f\n",
    "    y_pred = run_5fold(x_train, y_train, x_test, \"loss\", get_CNN_model, **params)\n",
    "    predictions_twitter[f] = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: 100\n",
      "loss: 0.04546199066764082 (+/- 0.001634462343494365)\n",
      "acc: 0.9830023201919488 (+/- 0.0004395332049064285)\n",
      "roc_auc: 0.9852713829746189 (+/- 0.0009901750425443203)\n",
      "\n",
      "\n",
      "\n",
      "key: 300\n",
      "loss: 0.04588088472710787 (+/- 0.002587987363538152)\n",
      "acc: 0.9828487878298529 (+/- 0.0009820900745316948)\n",
      "roc_auc: 0.9867106593074004 (+/- 0.0006931558414526911)\n",
      "\n",
      "\n",
      "\n",
      "key: 500\n",
      "loss: 0.045338480186612004 (+/- 0.0012909304583345598)\n",
      "acc: 0.9829542779099537 (+/- 0.0004968131354375467)\n",
      "roc_auc: 0.9851644077428029 (+/- 0.0012832848194246622)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_results(predictions_twitter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### result: \n",
    "\n",
    "- f = 50\n",
    "\n",
    "loss: 0.045302061441733474 (+/- 0.0007931266631046235)\n",
    "\n",
    "acc: 0.9829229399625717 (+/- 0.0004347512370955112)\n",
    "\n",
    "roc_auc: 0.9839322949097496 (+/- 0.0010873170500473907)\n",
    "\n",
    "\n",
    "- f = 100\n",
    "\n",
    "loss: 0.04546199066764082 (+/- 0.001634462343494365)\n",
    "\n",
    "acc: 0.9830023201919488 (+/- 0.0004395332049064285)\n",
    "\n",
    "roc_auc: 0.9852713829746189 (+/- 0.0009901750425443203)\n",
    "\n",
    "- f = 300\n",
    "\n",
    "loss: 0.04588088472710787 (+/- 0.002587987363538152)\n",
    "\n",
    "acc: 0.9828487878298529 (+/- 0.0009820900745316948)\n",
    "\n",
    "roc_auc: 0.9867106593074004 (+/- 0.0006931558414526911)\n",
    "\n",
    "- f = 500\n",
    "\n",
    "loss: 0.045338480186612004 (+/- 0.0012909304583345598)\n",
    "\n",
    "acc: 0.9829542779099537 (+/- 0.0004968131354375467)\n",
    "\n",
    "roc_auc: 0.9851644077428029 (+/- 0.0012832848194246622)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_twitter = predictions['glove_twitter']['pred']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check kernel_size\n",
    "(suggest value: [3, 5, 7, 9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"dropout\": 0.2,\n",
    "    \"num_filter\": 50,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove_twitter,\n",
    "    \"embed_size\": embed_size_glove_twitter\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "kernels = [5, 7, 9]  # for 3, see result above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "********************k = 5********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 24s - loss: 0.0649 - acc: 0.9776 - auc_roc: 0.9187 - val_loss: 0.0477 - val_acc: 0.9823 - val_auc_roc: 0.9696\n",
      "Epoch 2/50\n",
      " - 24s - loss: 0.0433 - acc: 0.9838 - auc_roc: 0.9761 - val_loss: 0.0454 - val_acc: 0.9828 - val_auc_roc: 0.9798\n",
      "Epoch 3/50\n",
      " - 24s - loss: 0.0372 - acc: 0.9858 - auc_roc: 0.9821 - val_loss: 0.0451 - val_acc: 0.9829 - val_auc_roc: 0.9837\n",
      "Epoch 4/50\n",
      " - 24s - loss: 0.0321 - acc: 0.9875 - auc_roc: 0.9851 - val_loss: 0.0475 - val_acc: 0.9829 - val_auc_roc: 0.9859\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04753291173478419\n",
      "val acc: 0.9828868136106381\n",
      "val auc roc: 0.9858642352574226\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 24s - loss: 0.0654 - acc: 0.9778 - auc_roc: 0.9087 - val_loss: 0.0485 - val_acc: 0.9821 - val_auc_roc: 0.9686\n",
      "Epoch 2/50\n",
      " - 24s - loss: 0.0430 - acc: 0.9839 - auc_roc: 0.9756 - val_loss: 0.0470 - val_acc: 0.9823 - val_auc_roc: 0.9792\n",
      "Epoch 3/50\n",
      " - 24s - loss: 0.0368 - acc: 0.9858 - auc_roc: 0.9815 - val_loss: 0.0455 - val_acc: 0.9830 - val_auc_roc: 0.9831\n",
      "Epoch 4/50\n",
      " - 24s - loss: 0.0319 - acc: 0.9877 - auc_roc: 0.9845 - val_loss: 0.0469 - val_acc: 0.9825 - val_auc_roc: 0.9855\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04685237737051629\n",
      "val acc: 0.9825102687442319\n",
      "val auc roc: 0.985522902791808\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 24s - loss: 0.0678 - acc: 0.9767 - auc_roc: 0.9077 - val_loss: 0.0459 - val_acc: 0.9828 - val_auc_roc: 0.9678\n",
      "Epoch 2/50\n",
      " - 24s - loss: 0.0433 - acc: 0.9836 - auc_roc: 0.9755 - val_loss: 0.0439 - val_acc: 0.9834 - val_auc_roc: 0.9793\n",
      "Epoch 3/50\n",
      " - 24s - loss: 0.0373 - acc: 0.9855 - auc_roc: 0.9818 - val_loss: 0.0447 - val_acc: 0.9835 - val_auc_roc: 0.9834\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.044741609155900156\n",
      "val acc: 0.9834555135287535\n",
      "val auc roc: 0.983409090475263\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 24s - loss: 0.0680 - acc: 0.9766 - auc_roc: 0.9093 - val_loss: 0.0474 - val_acc: 0.9826 - val_auc_roc: 0.9670\n",
      "Epoch 2/50\n",
      " - 24s - loss: 0.0439 - acc: 0.9836 - auc_roc: 0.9745 - val_loss: 0.0450 - val_acc: 0.9829 - val_auc_roc: 0.9784\n",
      "Epoch 3/50\n",
      " - 24s - loss: 0.0374 - acc: 0.9856 - auc_roc: 0.9811 - val_loss: 0.0458 - val_acc: 0.9828 - val_auc_roc: 0.9828\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04584289073734913\n",
      "val acc: 0.9827661593994147\n",
      "val auc roc: 0.9828161727540168\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 24s - loss: 0.0611 - acc: 0.9792 - auc_roc: 0.9262 - val_loss: 0.0467 - val_acc: 0.9825 - val_auc_roc: 0.9724\n",
      "Epoch 2/50\n",
      " - 24s - loss: 0.0421 - acc: 0.9840 - auc_roc: 0.9784 - val_loss: 0.0452 - val_acc: 0.9829 - val_auc_roc: 0.9813\n",
      "Epoch 3/50\n",
      " - 24s - loss: 0.0364 - acc: 0.9858 - auc_roc: 0.9834 - val_loss: 0.0453 - val_acc: 0.9830 - val_auc_roc: 0.9847\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04534178615415287\n",
      "val acc: 0.9829646088227283\n",
      "val auc roc: 0.9846961618463385\n",
      "\n",
      "\n",
      "********************k = 7********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0671 - acc: 0.9769 - auc_roc: 0.9071 - val_loss: 0.0480 - val_acc: 0.9821 - val_auc_roc: 0.9670\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0431 - acc: 0.9838 - auc_roc: 0.9747 - val_loss: 0.0455 - val_acc: 0.9826 - val_auc_roc: 0.9786\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0362 - acc: 0.9861 - auc_roc: 0.9812 - val_loss: 0.0463 - val_acc: 0.9827 - val_auc_roc: 0.9830\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04628858982188359\n",
      "val acc: 0.9827301471449637\n",
      "val auc roc: 0.9830090046937462\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0622 - acc: 0.9786 - auc_roc: 0.9229 - val_loss: 0.0483 - val_acc: 0.9824 - val_auc_roc: 0.9710\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0424 - acc: 0.9840 - auc_roc: 0.9771 - val_loss: 0.0455 - val_acc: 0.9832 - val_auc_roc: 0.9805\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0360 - acc: 0.9863 - auc_roc: 0.9827 - val_loss: 0.0455 - val_acc: 0.9829 - val_auc_roc: 0.9842\n",
      "Epoch 4/50\n",
      " - 26s - loss: 0.0305 - acc: 0.9883 - auc_roc: 0.9854 - val_loss: 0.0478 - val_acc: 0.9825 - val_auc_roc: 0.9863\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.047841342639635846\n",
      "val acc: 0.9825468243472056\n",
      "val auc roc: 0.9863383003125971\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0655 - acc: 0.9779 - auc_roc: 0.9138 - val_loss: 0.0463 - val_acc: 0.9829 - val_auc_roc: 0.9699\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0432 - acc: 0.9838 - auc_roc: 0.9763 - val_loss: 0.0438 - val_acc: 0.9836 - val_auc_roc: 0.9800\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0368 - acc: 0.9858 - auc_roc: 0.9824 - val_loss: 0.0441 - val_acc: 0.9834 - val_auc_roc: 0.9839\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04411534362623116\n",
      "val acc: 0.9834294035193122\n",
      "val auc roc: 0.9838753237553675\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0721 - acc: 0.9746 - auc_roc: 0.8943 - val_loss: 0.0472 - val_acc: 0.9826 - val_auc_roc: 0.9637\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0434 - acc: 0.9837 - auc_roc: 0.9728 - val_loss: 0.0449 - val_acc: 0.9831 - val_auc_roc: 0.9773\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0370 - acc: 0.9858 - auc_roc: 0.9802 - val_loss: 0.0453 - val_acc: 0.9832 - val_auc_roc: 0.9820\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.045333334106288696\n",
      "val acc: 0.9832413964024416\n",
      "val auc roc: 0.9819667935670225\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 26s - loss: 0.0653 - acc: 0.9779 - auc_roc: 0.9106 - val_loss: 0.0485 - val_acc: 0.9820 - val_auc_roc: 0.9676\n",
      "Epoch 2/50\n",
      " - 26s - loss: 0.0427 - acc: 0.9839 - auc_roc: 0.9752 - val_loss: 0.0466 - val_acc: 0.9825 - val_auc_roc: 0.9790\n",
      "Epoch 3/50\n",
      " - 26s - loss: 0.0363 - acc: 0.9861 - auc_roc: 0.9815 - val_loss: 0.0457 - val_acc: 0.9830 - val_auc_roc: 0.9830\n",
      "Epoch 4/50\n",
      " - 26s - loss: 0.0306 - acc: 0.9882 - auc_roc: 0.9845 - val_loss: 0.0479 - val_acc: 0.9827 - val_auc_roc: 0.9854\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.047902801907707994\n",
      "val acc: 0.9827452733110738\n",
      "val auc roc: 0.9854077382547004\n",
      "\n",
      "\n",
      "********************k = 9********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 28s - loss: 0.0656 - acc: 0.9776 - auc_roc: 0.9089 - val_loss: 0.0472 - val_acc: 0.9822 - val_auc_roc: 0.9667\n",
      "Epoch 2/50\n",
      " - 28s - loss: 0.0424 - acc: 0.9841 - auc_roc: 0.9746 - val_loss: 0.0467 - val_acc: 0.9825 - val_auc_roc: 0.9784\n",
      "Epoch 3/50\n",
      " - 28s - loss: 0.0356 - acc: 0.9864 - auc_roc: 0.9811 - val_loss: 0.0469 - val_acc: 0.9827 - val_auc_roc: 0.9829\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04694334225288178\n",
      "val acc: 0.9826674842804591\n",
      "val auc roc: 0.9829085636900899\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 28s - loss: 0.0660 - acc: 0.9772 - auc_roc: 0.9121 - val_loss: 0.0483 - val_acc: 0.9822 - val_auc_roc: 0.9687\n",
      "Epoch 2/50\n",
      " - 28s - loss: 0.0430 - acc: 0.9840 - auc_roc: 0.9756 - val_loss: 0.0475 - val_acc: 0.9822 - val_auc_roc: 0.9791\n",
      "Epoch 3/50\n",
      " - 28s - loss: 0.0361 - acc: 0.9863 - auc_roc: 0.9814 - val_loss: 0.0460 - val_acc: 0.9828 - val_auc_roc: 0.9831\n",
      "Epoch 4/50\n",
      " - 28s - loss: 0.0301 - acc: 0.9885 - auc_roc: 0.9845 - val_loss: 0.0479 - val_acc: 0.9827 - val_auc_roc: 0.9856\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04794804377126135\n",
      "val acc: 0.9826982722154213\n",
      "val auc roc: 0.9856033114522758\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 28s - loss: 0.0664 - acc: 0.9773 - auc_roc: 0.9107 - val_loss: 0.0469 - val_acc: 0.9825 - val_auc_roc: 0.9675\n",
      "Epoch 2/50\n",
      " - 28s - loss: 0.0430 - acc: 0.9839 - auc_roc: 0.9749 - val_loss: 0.0444 - val_acc: 0.9834 - val_auc_roc: 0.9788\n",
      "Epoch 3/50\n",
      " - 28s - loss: 0.0359 - acc: 0.9862 - auc_roc: 0.9815 - val_loss: 0.0452 - val_acc: 0.9833 - val_auc_roc: 0.9831\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04516441798416708\n",
      "val acc: 0.9832779547546175\n",
      "val auc roc: 0.9830831436762171\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 28s - loss: 0.0674 - acc: 0.9773 - auc_roc: 0.9056 - val_loss: 0.0466 - val_acc: 0.9828 - val_auc_roc: 0.9660\n",
      "Epoch 2/50\n",
      " - 28s - loss: 0.0426 - acc: 0.9840 - auc_roc: 0.9741 - val_loss: 0.0455 - val_acc: 0.9827 - val_auc_roc: 0.9785\n",
      "Epoch 3/50\n",
      " - 28s - loss: 0.0355 - acc: 0.9864 - auc_roc: 0.9813 - val_loss: 0.0468 - val_acc: 0.9824 - val_auc_roc: 0.9831\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00003: early stopping\n",
      "val loss: 0.046753579186505675\n",
      "val acc: 0.9823588156166729\n",
      "val auc roc: 0.9831283277883157\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 28s - loss: 0.0635 - acc: 0.9783 - auc_roc: 0.9198 - val_loss: 0.0482 - val_acc: 0.9820 - val_auc_roc: 0.9701\n",
      "Epoch 2/50\n",
      " - 28s - loss: 0.0422 - acc: 0.9841 - auc_roc: 0.9767 - val_loss: 0.0456 - val_acc: 0.9829 - val_auc_roc: 0.9801\n",
      "Epoch 3/50\n",
      " - 28s - loss: 0.0352 - acc: 0.9865 - auc_roc: 0.9825 - val_loss: 0.0469 - val_acc: 0.9828 - val_auc_roc: 0.9839\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04687114304075521\n",
      "val acc: 0.9828497244651779\n",
      "val auc roc: 0.9838933663319815\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for k in kernels:\n",
    "    print (\"\\n********************k = {}********************\".format(k))\n",
    "    params[\"kernel_size\"] = k\n",
    "    result = run_5fold(x_train, y_train, x_test, \"loss\", get_CNN_model, **params)    \n",
    "    predictions_twitter[k] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: 100\n",
      "loss: 0.04546199066764082 (+/- 0.001634462343494365)\n",
      "acc: 0.9830023201919488 (+/- 0.0004395332049064285)\n",
      "roc_auc: 0.9852713829746189 (+/- 0.0009901750425443203)\n",
      "\n",
      "\n",
      "\n",
      "key: 300\n",
      "loss: 0.04588088472710787 (+/- 0.002587987363538152)\n",
      "acc: 0.9828487878298529 (+/- 0.0009820900745316948)\n",
      "roc_auc: 0.9867106593074004 (+/- 0.0006931558414526911)\n",
      "\n",
      "\n",
      "\n",
      "key: 500\n",
      "loss: 0.045338480186612004 (+/- 0.0012909304583345598)\n",
      "acc: 0.9829542779099537 (+/- 0.0004968131354375467)\n",
      "roc_auc: 0.9851644077428029 (+/- 0.0012832848194246622)\n",
      "\n",
      "\n",
      "\n",
      "key: 5\n",
      "loss: 0.04606231503054052 (+/- 0.0010097946382350116)\n",
      "acc: 0.9829166728211532 (+/- 0.0003102764266677338)\n",
      "roc_auc: 0.9844617126249698 (+/- 0.001180167411775649)\n",
      "\n",
      "\n",
      "\n",
      "key: 7\n",
      "loss: 0.04629628242034946 (+/- 0.0014595847765578977)\n",
      "acc: 0.9829386089449994 (+/- 0.0003367164145233068)\n",
      "roc_auc: 0.9841194321166867 (+/- 0.0015817458364257356)\n",
      "\n",
      "\n",
      "\n",
      "key: 9\n",
      "loss: 0.04673610524711422 (+/- 0.0008944812529786167)\n",
      "acc: 0.9827704502664698 (+/- 0.0002996998346054342)\n",
      "roc_auc: 0.9837233425877759 (+/- 0.0009990948818940741)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_results(predictions_twitter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### result:\n",
    "\n",
    "- k = 3\n",
    "\n",
    "loss: 0.045302061441733474 (+/- 0.0007931266631046235)\n",
    "\n",
    "acc: 0.9829229399625717 (+/- 0.0004347512370955112)\n",
    "\n",
    "roc_auc: 0.9839322949097496 (+/- 0.0010873170500473907)\n",
    "\n",
    "- k = 5\n",
    "\n",
    "loss: 0.04606231503054052 (+/- 0.0010097946382350116)\n",
    "\n",
    "acc: 0.9829166728211532 (+/- 0.0003102764266677338)\n",
    "\n",
    "roc_auc: 0.9844617126249698 (+/- 0.001180167411775649)\n",
    "\n",
    "- k = 7\n",
    "\n",
    "loss: 0.04629628242034946 (+/- 0.0014595847765578977)\n",
    "\n",
    "acc: 0.9829386089449994 (+/- 0.0003367164145233068)\n",
    "\n",
    "roc_auc: 0.9841194321166867 (+/- 0.0015817458364257356)\n",
    "\n",
    "\n",
    "- k = 9\n",
    "\n",
    "loss: 0.04673610524711422 (+/- 0.0008944812529786167)\n",
    "\n",
    "acc: 0.9827704502664698 (+/- 0.0002996998346054342)\n",
    "\n",
    "roc_auc: 0.9837233425877759 (+/- 0.0009990948818940741)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_twitter = opt_twitter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check drop out\n",
    "(suggest value 0.1~0.5: test 0.1, 0.2, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"kernel_size\": 3,\n",
    "    \"num_filter\": 50,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove_twitter,\n",
    "    \"embed_size\": embed_size_glove_twitter\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropouts = [0.1, 0.5] # 0.2 see previous result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "********************d = 0.1********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 22s - loss: 0.0614 - acc: 0.9788 - auc_roc: 0.9213 - val_loss: 0.0465 - val_acc: 0.9823 - val_auc_roc: 0.9741\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0415 - acc: 0.9842 - auc_roc: 0.9797 - val_loss: 0.0447 - val_acc: 0.9830 - val_auc_roc: 0.9825\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0353 - acc: 0.9863 - auc_roc: 0.9844 - val_loss: 0.0456 - val_acc: 0.9826 - val_auc_roc: 0.9857\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04559372121892191\n",
      "val acc: 0.9826361501700598\n",
      "val auc roc: 0.9857020561642432\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0700 - acc: 0.9748 - auc_roc: 0.8973 - val_loss: 0.0473 - val_acc: 0.9824 - val_auc_roc: 0.9681\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0424 - acc: 0.9840 - auc_roc: 0.9755 - val_loss: 0.0454 - val_acc: 0.9828 - val_auc_roc: 0.9796\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0361 - acc: 0.9862 - auc_roc: 0.9821 - val_loss: 0.0456 - val_acc: 0.9832 - val_auc_roc: 0.9838\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.045647551806301236\n",
      "val acc: 0.983163059041667\n",
      "val auc roc: 0.9837613555283417\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0650 - acc: 0.9778 - auc_roc: 0.9056 - val_loss: 0.0461 - val_acc: 0.9829 - val_auc_roc: 0.9695\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0425 - acc: 0.9838 - auc_roc: 0.9767 - val_loss: 0.0446 - val_acc: 0.9832 - val_auc_roc: 0.9803\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0362 - acc: 0.9859 - auc_roc: 0.9827 - val_loss: 0.0445 - val_acc: 0.9836 - val_auc_roc: 0.9842\n",
      "Epoch 4/50\n",
      " - 21s - loss: 0.0309 - acc: 0.9879 - auc_roc: 0.9856 - val_loss: 0.0459 - val_acc: 0.9831 - val_auc_roc: 0.9865\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04591641086960575\n",
      "val acc: 0.9830795080207408\n",
      "val auc roc: 0.986498892142927\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0655 - acc: 0.9770 - auc_roc: 0.9087 - val_loss: 0.0466 - val_acc: 0.9828 - val_auc_roc: 0.9698\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0423 - acc: 0.9841 - auc_roc: 0.9769 - val_loss: 0.0443 - val_acc: 0.9833 - val_auc_roc: 0.9803\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0359 - acc: 0.9861 - auc_roc: 0.9827 - val_loss: 0.0449 - val_acc: 0.9831 - val_auc_roc: 0.9843\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.044898685159174175\n",
      "val acc: 0.9830795065863744\n",
      "val auc roc: 0.9843174561885258\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0732 - acc: 0.9754 - auc_roc: 0.8879 - val_loss: 0.0486 - val_acc: 0.9820 - val_auc_roc: 0.9628\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0430 - acc: 0.9838 - auc_roc: 0.9723 - val_loss: 0.0459 - val_acc: 0.9823 - val_auc_roc: 0.9772\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0362 - acc: 0.9859 - auc_roc: 0.9802 - val_loss: 0.0457 - val_acc: 0.9826 - val_auc_roc: 0.9823\n",
      "Epoch 4/50\n",
      " - 21s - loss: 0.0305 - acc: 0.9882 - auc_roc: 0.9840 - val_loss: 0.0475 - val_acc: 0.9821 - val_auc_roc: 0.9851\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04752270146917713\n",
      "val acc: 0.9820506969340621\n",
      "val auc roc: 0.9851387091027232\n",
      "\n",
      "\n",
      "********************d = 0.5********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0757 - acc: 0.9750 - auc_roc: 0.8861 - val_loss: 0.0506 - val_acc: 0.9817 - val_auc_roc: 0.9553\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0495 - acc: 0.9820 - auc_roc: 0.9656 - val_loss: 0.0468 - val_acc: 0.9824 - val_auc_roc: 0.9715\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0448 - acc: 0.9833 - auc_roc: 0.9749 - val_loss: 0.0474 - val_acc: 0.9819 - val_auc_roc: 0.9772\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.047363344746035284\n",
      "val acc: 0.9818580384652629\n",
      "val auc roc: 0.9771679343688681\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0840 - acc: 0.9707 - auc_roc: 0.8649 - val_loss: 0.0510 - val_acc: 0.9815 - val_auc_roc: 0.9524\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0494 - acc: 0.9820 - auc_roc: 0.9639 - val_loss: 0.0462 - val_acc: 0.9827 - val_auc_roc: 0.9702\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0452 - acc: 0.9832 - auc_roc: 0.9737 - val_loss: 0.0462 - val_acc: 0.9826 - val_auc_roc: 0.9763\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04623064693132835\n",
      "val acc: 0.9826460452040029\n",
      "val auc roc: 0.976268945634085\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0727 - acc: 0.9763 - auc_roc: 0.8855 - val_loss: 0.0482 - val_acc: 0.9825 - val_auc_roc: 0.9584\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0495 - acc: 0.9820 - auc_roc: 0.9678 - val_loss: 0.0454 - val_acc: 0.9832 - val_auc_roc: 0.9728\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0453 - acc: 0.9830 - auc_roc: 0.9759 - val_loss: 0.0457 - val_acc: 0.9830 - val_auc_roc: 0.9780\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.0456814075772114\n",
      "val acc: 0.9829959466193354\n",
      "val auc roc: 0.9780225945145085\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0748 - acc: 0.9752 - auc_roc: 0.8806 - val_loss: 0.0504 - val_acc: 0.9814 - val_auc_roc: 0.9568\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0491 - acc: 0.9819 - auc_roc: 0.9670 - val_loss: 0.0459 - val_acc: 0.9827 - val_auc_roc: 0.9723\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0449 - acc: 0.9832 - auc_roc: 0.9754 - val_loss: 0.0452 - val_acc: 0.9827 - val_auc_roc: 0.9775\n",
      "Epoch 4/50\n",
      " - 21s - loss: 0.0420 - acc: 0.9840 - auc_roc: 0.9792 - val_loss: 0.0451 - val_acc: 0.9829 - val_auc_roc: 0.9805\n",
      "Epoch 5/50\n",
      " - 21s - loss: 0.0397 - acc: 0.9848 - auc_roc: 0.9815 - val_loss: 0.0438 - val_acc: 0.9834 - val_auc_roc: 0.9823\n",
      "Epoch 6/50\n",
      " - 21s - loss: 0.0378 - acc: 0.9855 - auc_roc: 0.9830 - val_loss: 0.0449 - val_acc: 0.9829 - val_auc_roc: 0.9836\n",
      "Epoch 00006: early stopping\n",
      "val loss: 0.044896900494620716\n",
      "val acc: 0.9829123856960521\n",
      "val auc roc: 0.983611492791809\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 21s - loss: 0.0780 - acc: 0.9739 - auc_roc: 0.8716 - val_loss: 0.0512 - val_acc: 0.9814 - val_auc_roc: 0.9546\n",
      "Epoch 2/50\n",
      " - 21s - loss: 0.0496 - acc: 0.9818 - auc_roc: 0.9654 - val_loss: 0.0472 - val_acc: 0.9821 - val_auc_roc: 0.9712\n",
      "Epoch 3/50\n",
      " - 21s - loss: 0.0453 - acc: 0.9830 - auc_roc: 0.9746 - val_loss: 0.0465 - val_acc: 0.9825 - val_auc_roc: 0.9769\n",
      "Epoch 4/50\n",
      " - 21s - loss: 0.0422 - acc: 0.9840 - auc_roc: 0.9787 - val_loss: 0.0468 - val_acc: 0.9820 - val_auc_roc: 0.9800\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.0468026816745954\n",
      "val acc: 0.9820193620061878\n",
      "val auc roc: 0.9800491874032864\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for d in dropouts:\n",
    "    print (\"\\n********************d = {}********************\".format(d))\n",
    "    params[\"dropout\"] = d\n",
    "    result = run_5fold(x_train, y_train, x_test, \"loss\", get_CNN_model, **params)    \n",
    "    predictions_twitter[d] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: 100\n",
      "loss: 0.04546199066764082 (+/- 0.001634462343494365)\n",
      "acc: 0.9830023201919488 (+/- 0.0004395332049064285)\n",
      "roc_auc: 0.9852713829746189 (+/- 0.0009901750425443203)\n",
      "\n",
      "\n",
      "\n",
      "key: 300\n",
      "loss: 0.04588088472710787 (+/- 0.002587987363538152)\n",
      "acc: 0.9828487878298529 (+/- 0.0009820900745316948)\n",
      "roc_auc: 0.9867106593074004 (+/- 0.0006931558414526911)\n",
      "\n",
      "\n",
      "\n",
      "key: 500\n",
      "loss: 0.045338480186612004 (+/- 0.0012909304583345598)\n",
      "acc: 0.9829542779099537 (+/- 0.0004968131354375467)\n",
      "roc_auc: 0.9851644077428029 (+/- 0.0012832848194246622)\n",
      "\n",
      "\n",
      "\n",
      "key: 5\n",
      "loss: 0.04606231503054052 (+/- 0.0010097946382350116)\n",
      "acc: 0.9829166728211532 (+/- 0.0003102764266677338)\n",
      "roc_auc: 0.9844617126249698 (+/- 0.001180167411775649)\n",
      "\n",
      "\n",
      "\n",
      "key: 7\n",
      "loss: 0.04629628242034946 (+/- 0.0014595847765578977)\n",
      "acc: 0.9829386089449994 (+/- 0.0003367164145233068)\n",
      "roc_auc: 0.9841194321166867 (+/- 0.0015817458364257356)\n",
      "\n",
      "\n",
      "\n",
      "key: 9\n",
      "loss: 0.04673610524711422 (+/- 0.0008944812529786167)\n",
      "acc: 0.9827704502664698 (+/- 0.0002996998346054342)\n",
      "roc_auc: 0.9837233425877759 (+/- 0.0009990948818940741)\n",
      "\n",
      "\n",
      "\n",
      "key: 0.1\n",
      "loss: 0.04591581410463604 (+/- 0.0008709015027983505)\n",
      "acc: 0.9828017841505808 (+/- 0.00041865175710850976)\n",
      "roc_auc: 0.9850836938253522 (+/- 0.0009719962580997339)\n",
      "\n",
      "\n",
      "\n",
      "key: 0.5\n",
      "loss: 0.046194996284758226 (+/- 0.0008584190898952846)\n",
      "acc: 0.9824863555981682 (+/- 0.0004646613514855763)\n",
      "roc_auc: 0.9790240309425113 (+/- 0.002612822858693605)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_results(predictions_twitter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### result\n",
    "\n",
    "- d = 0.1\n",
    "\n",
    "loss: 0.04591581410463604 (+/- 0.0008709015027983505)\n",
    "\n",
    "acc: 0.9828017841505808 (+/- 0.00041865175710850976)\n",
    "\n",
    "roc_auc: 0.9850836938253522 (+/- 0.0009719962580997339)\n",
    "\n",
    "- d = 0.2\n",
    "\n",
    "loss: 0.045302061441733474 (+/- 0.0007931266631046235)\n",
    "\n",
    "acc: 0.9829229399625717 (+/- 0.0004347512370955112)\n",
    "\n",
    "roc_auc: 0.9839322949097496 (+/- 0.0010873170500473907)\n",
    "\n",
    "\n",
    "- d = 0.5\n",
    "\n",
    "loss: 0.046194996284758226 (+/- 0.0008584190898952846)\n",
    "\n",
    "acc: 0.9824863555981682 (+/- 0.0004646613514855763)\n",
    "\n",
    "roc_auc: 0.9790240309425113 (+/- 0.002612822858693605)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_twitter = opt_twitter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### result params for glove twitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"kernel_size\": 3,\n",
    "    \"num_filter\": 50,\n",
    "    \"dropout\": 0.2,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove_twitter,\n",
    "    \"embed_size\": embed_size_glove_twitter\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_prediction(opt_twitter, submission, \"output/CNN_twitter.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# single model for glove"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check num_filter\n",
    "(suggest values: 100~600: test 100, 300, 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = [100, 300, 500]  # for the result of 50, refer to the previous log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"dropout\": 0.2,\n",
    "    \"kernel_size\": 3,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove,\n",
    "    \"embed_size\": embed_size_glove\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_glove = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "********************f = 100********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 33s - loss: 0.0579 - acc: 0.9797 - auc_roc: 0.9289 - val_loss: 0.0451 - val_acc: 0.9828 - val_auc_roc: 0.9755\n",
      "Epoch 2/50\n",
      " - 33s - loss: 0.0401 - acc: 0.9848 - auc_roc: 0.9805 - val_loss: 0.0449 - val_acc: 0.9827 - val_auc_roc: 0.9830\n",
      "Epoch 3/50\n",
      " - 33s - loss: 0.0338 - acc: 0.9870 - auc_roc: 0.9850 - val_loss: 0.0447 - val_acc: 0.9830 - val_auc_roc: 0.9861\n",
      "Epoch 4/50\n",
      " - 33s - loss: 0.0275 - acc: 0.9894 - auc_roc: 0.9873 - val_loss: 0.0498 - val_acc: 0.9815 - val_auc_roc: 0.9881\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.04975917948724782\n",
      "val acc: 0.9815394875604718\n",
      "val auc roc: 0.988082192087181\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 33s - loss: 0.0595 - acc: 0.9789 - auc_roc: 0.9235 - val_loss: 0.0452 - val_acc: 0.9827 - val_auc_roc: 0.9737\n",
      "Epoch 2/50\n",
      " - 33s - loss: 0.0403 - acc: 0.9845 - auc_roc: 0.9794 - val_loss: 0.0446 - val_acc: 0.9829 - val_auc_roc: 0.9823\n",
      "Epoch 3/50\n",
      " - 33s - loss: 0.0339 - acc: 0.9869 - auc_roc: 0.9842 - val_loss: 0.0452 - val_acc: 0.9831 - val_auc_roc: 0.9856\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04522530258167856\n",
      "val acc: 0.9830586144617418\n",
      "val auc roc: 0.9855594743296954\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 33s - loss: 0.0570 - acc: 0.9796 - auc_roc: 0.9354 - val_loss: 0.0435 - val_acc: 0.9834 - val_auc_roc: 0.9767\n",
      "Epoch 2/50\n",
      " - 33s - loss: 0.0404 - acc: 0.9845 - auc_roc: 0.9812 - val_loss: 0.0425 - val_acc: 0.9836 - val_auc_roc: 0.9836\n",
      "Epoch 3/50\n",
      " - 33s - loss: 0.0337 - acc: 0.9868 - auc_roc: 0.9854 - val_loss: 0.0446 - val_acc: 0.9833 - val_auc_roc: 0.9864\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04455429571250286\n",
      "val acc: 0.9832779582807681\n",
      "val auc roc: 0.9863998966310573\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 33s - loss: 0.0575 - acc: 0.9793 - auc_roc: 0.9351 - val_loss: 0.0451 - val_acc: 0.9832 - val_auc_roc: 0.9775\n",
      "Epoch 2/50\n",
      " - 33s - loss: 0.0402 - acc: 0.9845 - auc_roc: 0.9818 - val_loss: 0.0434 - val_acc: 0.9833 - val_auc_roc: 0.9841\n",
      "Epoch 3/50\n",
      " - 33s - loss: 0.0336 - acc: 0.9869 - auc_roc: 0.9859 - val_loss: 0.0442 - val_acc: 0.9833 - val_auc_roc: 0.9869\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.044209082989515786\n",
      "val acc: 0.9832936213220758\n",
      "val auc roc: 0.9869138242726697\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 33s - loss: 0.0594 - acc: 0.9792 - auc_roc: 0.9279 - val_loss: 0.0457 - val_acc: 0.9829 - val_auc_roc: 0.9746\n",
      "Epoch 2/50\n",
      " - 33s - loss: 0.0403 - acc: 0.9844 - auc_roc: 0.9798 - val_loss: 0.0446 - val_acc: 0.9829 - val_auc_roc: 0.9826\n",
      "Epoch 3/50\n",
      " - 33s - loss: 0.0340 - acc: 0.9867 - auc_roc: 0.9845 - val_loss: 0.0463 - val_acc: 0.9827 - val_auc_roc: 0.9857\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04632442073761472\n",
      "val acc: 0.9827452736509887\n",
      "val auc roc: 0.9856723158843775\n",
      "\n",
      "\n",
      "********************f = 300********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0548 - acc: 0.9803 - auc_roc: 0.9416 - val_loss: 0.0473 - val_acc: 0.9818 - val_auc_roc: 0.9784\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0389 - acc: 0.9851 - auc_roc: 0.9825 - val_loss: 0.0448 - val_acc: 0.9833 - val_auc_roc: 0.9846\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0318 - acc: 0.9877 - auc_roc: 0.9862 - val_loss: 0.0464 - val_acc: 0.9826 - val_auc_roc: 0.9872\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04638249798488621\n",
      "val acc: 0.9825995923489328\n",
      "val auc roc: 0.9872262479648981\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0541 - acc: 0.9806 - auc_roc: 0.9446 - val_loss: 0.0448 - val_acc: 0.9832 - val_auc_roc: 0.9793\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0388 - acc: 0.9851 - auc_roc: 0.9828 - val_loss: 0.0434 - val_acc: 0.9833 - val_auc_roc: 0.9848\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0318 - acc: 0.9877 - auc_roc: 0.9864 - val_loss: 0.0455 - val_acc: 0.9829 - val_auc_roc: 0.9874\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04547208099562796\n",
      "val acc: 0.9828967186504717\n",
      "val auc roc: 0.9874366073259668\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0556 - acc: 0.9800 - auc_roc: 0.9386 - val_loss: 0.0434 - val_acc: 0.9835 - val_auc_roc: 0.9775\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0392 - acc: 0.9847 - auc_roc: 0.9819 - val_loss: 0.0446 - val_acc: 0.9832 - val_auc_roc: 0.9843\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.044617292129855976\n",
      "val acc: 0.9831526154614768\n",
      "val auc roc: 0.9842605069181618\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0583 - acc: 0.9788 - auc_roc: 0.9312 - val_loss: 0.0436 - val_acc: 0.9835 - val_auc_roc: 0.9772\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0393 - acc: 0.9849 - auc_roc: 0.9816 - val_loss: 0.0441 - val_acc: 0.9834 - val_auc_roc: 0.9838\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.04405917994878328\n",
      "val acc: 0.9833667362334697\n",
      "val auc roc: 0.9837983442539184\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0544 - acc: 0.9803 - auc_roc: 0.9447 - val_loss: 0.0436 - val_acc: 0.9834 - val_auc_roc: 0.9791\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0390 - acc: 0.9849 - auc_roc: 0.9830 - val_loss: 0.0441 - val_acc: 0.9832 - val_auc_roc: 0.9848\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.04413799279219242\n",
      "val acc: 0.9831891746690431\n",
      "val auc roc: 0.9848491491258103\n",
      "\n",
      "\n",
      "********************f = 500********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 71s - loss: 0.0529 - acc: 0.9810 - auc_roc: 0.9488 - val_loss: 0.0433 - val_acc: 0.9831 - val_auc_roc: 0.9795\n",
      "Epoch 2/50\n",
      " - 71s - loss: 0.0383 - acc: 0.9852 - auc_roc: 0.9833 - val_loss: 0.0470 - val_acc: 0.9823 - val_auc_roc: 0.9847\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.04697134000682988\n",
      "val acc: 0.9823175921721012\n",
      "val auc roc: 0.9846775107411369\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 71s - loss: 0.0532 - acc: 0.9809 - auc_roc: 0.9490 - val_loss: 0.0434 - val_acc: 0.9833 - val_auc_roc: 0.9797\n",
      "Epoch 2/50\n",
      " - 71s - loss: 0.0384 - acc: 0.9851 - auc_roc: 0.9833 - val_loss: 0.0444 - val_acc: 0.9831 - val_auc_roc: 0.9851\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.044429378356989326\n",
      "val acc: 0.9830690613514336\n",
      "val auc roc: 0.9850695613124297\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 71s - loss: 0.0535 - acc: 0.9806 - auc_roc: 0.9507 - val_loss: 0.0452 - val_acc: 0.9826 - val_auc_roc: 0.9796\n",
      "Epoch 2/50\n",
      " - 71s - loss: 0.0387 - acc: 0.9851 - auc_roc: 0.9833 - val_loss: 0.0435 - val_acc: 0.9836 - val_auc_roc: 0.9851\n",
      "Epoch 3/50\n",
      " - 71s - loss: 0.0310 - acc: 0.9880 - auc_roc: 0.9867 - val_loss: 0.0489 - val_acc: 0.9831 - val_auc_roc: 0.9873\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04888977827569062\n",
      "val acc: 0.9830638382259583\n",
      "val auc roc: 0.9872646226488185\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 71s - loss: 0.0540 - acc: 0.9807 - auc_roc: 0.9444 - val_loss: 0.0430 - val_acc: 0.9837 - val_auc_roc: 0.9788\n",
      "Epoch 2/50\n",
      " - 71s - loss: 0.0386 - acc: 0.9852 - auc_roc: 0.9829 - val_loss: 0.0434 - val_acc: 0.9837 - val_auc_roc: 0.9847\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.043442633124628094\n",
      "val acc: 0.9837427470605908\n",
      "val auc roc: 0.9846840537325768\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 71s - loss: 0.0534 - acc: 0.9808 - auc_roc: 0.9458 - val_loss: 0.0460 - val_acc: 0.9830 - val_auc_roc: 0.9789\n",
      "Epoch 2/50\n",
      " - 71s - loss: 0.0387 - acc: 0.9851 - auc_roc: 0.9826 - val_loss: 0.0454 - val_acc: 0.9829 - val_auc_roc: 0.9844\n",
      "Epoch 3/50\n",
      " - 71s - loss: 0.0308 - acc: 0.9881 - auc_roc: 0.9860 - val_loss: 0.0483 - val_acc: 0.9828 - val_auc_roc: 0.9869\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.0483035647879429\n",
      "val acc: 0.9827557198010852\n",
      "val auc roc: 0.9868901424885483\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for f in filters:\n",
    "    print (\"\\n********************f = {}********************\".format(f))\n",
    "    params[\"num_filter\"] = f\n",
    "    result = run_5fold(x_train, y_train, x_test, \"loss\", get_CNN_model, **params)    \n",
    "    predictions_glove[f] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: 100\n",
      "loss: 0.04601445630171195 (+/- 0.0020066506932860306)\n",
      "acc: 0.9827829910552092 (+/- 0.0006525990545582435)\n",
      "roc_auc: 0.9865255406409961 (+/- 0.0009220619202532495)\n",
      "\n",
      "\n",
      "\n",
      "key: 300\n",
      "loss: 0.04493380877026916 (+/- 0.0008816936950137447)\n",
      "acc: 0.9830409674726788 (+/- 0.0002668991024840636)\n",
      "roc_auc: 0.9855141711177511 (+/- 0.0015221667521623432)\n",
      "\n",
      "\n",
      "\n",
      "key: 500\n",
      "loss: 0.046407338910416165 (+/- 0.00213438906316632)\n",
      "acc: 0.982989791722234 (+/- 0.00046590775908032697)\n",
      "roc_auc: 0.9857171781847021 (+/- 0.0011258855863419304)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_results(predictions_glove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### result:\n",
    "\n",
    "- 50:\n",
    "\n",
    "loss: 0.04559248425298761 (+/- 0.00066710734655799)\n",
    "\n",
    "acc: 0.9827965611517264 (+/- 0.0003516438407890546)\n",
    "\n",
    "roc_auc: 0.9848998829240934 (+/- 0.0006772049605117584)\n",
    "\n",
    "- 100:\n",
    "\n",
    "loss: 0.04601445630171195 (+/- 0.0020066506932860306)\n",
    "\n",
    "acc: 0.9827829910552092 (+/- 0.0006525990545582435)\n",
    "\n",
    "roc_auc: 0.9865255406409961 (+/- 0.0009220619202532495)\n",
    "\n",
    "- 300:\n",
    "\n",
    "loss: 0.04493380877026916 (+/- 0.0008816936950137447)\n",
    "\n",
    "acc: 0.9830409674726788 (+/- 0.0002668991024840636)\n",
    "\n",
    "roc_auc: 0.9855141711177511 (+/- 0.0015221667521623432)\n",
    "\n",
    "\n",
    "- 500:\n",
    "\n",
    "loss: 0.046407338910416165 (+/- 0.00213438906316632)\n",
    "\n",
    "acc: 0.982989791722234 (+/- 0.00046590775908032697)\n",
    "\n",
    "roc_auc: 0.9857171781847021 (+/- 0.0011258855863419304)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_glove = predictions_glove[300]['pred']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[9.9988878e-01, 3.2607320e-01, 9.9909186e-01, 4.2837597e-02,\n",
       "         9.7526181e-01, 5.0506383e-01],\n",
       "        [5.2075975e-05, 6.7775509e-06, 3.7139853e-05, 8.9647619e-06,\n",
       "         7.7980643e-05, 2.6870233e-05],\n",
       "        [5.8451872e-03, 4.8241992e-03, 1.0806988e-02, 3.0471815e-04,\n",
       "         6.2295264e-03, 1.6877786e-03],\n",
       "        ...,\n",
       "        [9.8656739e-05, 6.1778963e-05, 2.6817099e-04, 1.4880783e-04,\n",
       "         5.4808811e-04, 8.7899913e-05],\n",
       "        [7.9450890e-04, 1.9844156e-04, 5.7480810e-04, 4.4849576e-04,\n",
       "         2.3525646e-03, 5.2878205e-03],\n",
       "        [9.9707162e-01, 6.8134344e-03, 9.5658046e-01, 4.4497019e-03,\n",
       "         4.3048623e-01, 5.1593492e-03]], dtype=float32),\n",
       " array([[9.9936670e-01, 1.5884735e-01, 9.8224074e-01, 1.1089378e-02,\n",
       "         9.8304409e-01, 7.5670975e-01],\n",
       "        [8.7981585e-05, 2.0524496e-05, 7.4573647e-05, 4.7461276e-06,\n",
       "         2.5087112e-04, 2.5516289e-05],\n",
       "        [1.0546318e-02, 2.5208511e-03, 1.4501218e-03, 6.8583700e-05,\n",
       "         3.4691619e-03, 2.1256869e-04],\n",
       "        ...,\n",
       "        [2.4898100e-04, 8.7214350e-05, 3.0761198e-04, 5.3234551e-05,\n",
       "         2.7098248e-04, 6.6380060e-05],\n",
       "        [1.9628224e-03, 1.5061967e-04, 1.3332222e-04, 5.8773581e-05,\n",
       "         1.5710404e-03, 2.7187334e-03],\n",
       "        [9.7443277e-01, 1.0664597e-03, 8.3501214e-01, 4.0311631e-04,\n",
       "         1.4526296e-01, 4.4580218e-03]], dtype=float32),\n",
       " array([[9.99462306e-01, 4.99829113e-01, 9.94108081e-01, 1.10794725e-02,\n",
       "         9.58979964e-01, 8.54436994e-01],\n",
       "        [4.83013166e-04, 6.59303187e-05, 3.24975117e-04, 3.41070081e-05,\n",
       "         8.17830383e-04, 1.09076493e-04],\n",
       "        [3.27506452e-03, 2.75748852e-03, 7.32445670e-03, 1.72941523e-04,\n",
       "         3.91785568e-03, 8.85187241e-04],\n",
       "        ...,\n",
       "        [8.16167885e-05, 2.34560775e-05, 1.18528296e-04, 9.76325737e-06,\n",
       "         1.33053734e-04, 2.05588265e-04],\n",
       "        [6.26077875e-03, 9.27796878e-04, 5.85718837e-04, 1.65001242e-04,\n",
       "         1.70444325e-03, 2.57460754e-02],\n",
       "        [9.95285213e-01, 1.13114873e-02, 9.14264798e-01, 2.82611256e-03,\n",
       "         7.51523435e-01, 5.50059974e-02]], dtype=float32),\n",
       " array([[9.9924088e-01, 1.1976563e-01, 9.3372637e-01, 4.8608165e-02,\n",
       "         7.7087873e-01, 4.2457041e-01],\n",
       "        [6.6556910e-04, 1.2496447e-04, 7.0857088e-04, 9.4218551e-05,\n",
       "         5.9544790e-04, 1.7559214e-04],\n",
       "        [4.6926565e-03, 1.5948199e-03, 2.6780586e-03, 5.7020236e-04,\n",
       "         2.0672274e-03, 8.9033105e-04],\n",
       "        ...,\n",
       "        [7.7043305e-04, 3.8994687e-05, 7.6163313e-05, 5.8377769e-05,\n",
       "         9.0816422e-05, 6.1555729e-05],\n",
       "        [2.4165153e-03, 1.9376277e-04, 3.1315067e-04, 3.2345590e-04,\n",
       "         3.2720537e-04, 1.2767407e-02],\n",
       "        [9.9230754e-01, 1.6358061e-03, 6.9659007e-01, 1.3737958e-02,\n",
       "         2.1676567e-01, 6.9929129e-03]], dtype=float32),\n",
       " array([[9.9660468e-01, 4.3298805e-01, 9.9314779e-01, 4.2447232e-02,\n",
       "         7.9327106e-01, 3.4500161e-01],\n",
       "        [6.2479178e-04, 7.0764814e-05, 1.9084338e-04, 3.5651592e-05,\n",
       "         2.1260315e-04, 4.7497753e-05],\n",
       "        [6.2960424e-03, 3.0548789e-03, 1.3664165e-02, 7.5632753e-04,\n",
       "         5.5329157e-03, 5.2696984e-04],\n",
       "        ...,\n",
       "        [6.8543428e-05, 2.3263759e-05, 1.3622676e-04, 5.8052843e-05,\n",
       "         1.5244952e-04, 6.5511857e-05],\n",
       "        [6.7387027e-04, 2.1552734e-04, 1.1593502e-03, 1.6542958e-04,\n",
       "         7.2906126e-04, 7.2318665e-03],\n",
       "        [9.5108610e-01, 9.7425980e-03, 9.2711061e-01, 4.9974974e-03,\n",
       "         1.4178453e-01, 5.8298428e-03]], dtype=float32)]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt_glove"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check kernel size\n",
    "(suggest value: [3, 5, 7, 9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"dropout\": 0.2,\n",
    "    \"num_filter\": 300,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove,\n",
    "    \"embed_size\": embed_size_glove\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "kernels = [5, 7, 9]  # for 3, see result above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "********************k = 5********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 72s - loss: 0.0560 - acc: 0.9799 - auc_roc: 0.9400 - val_loss: 0.0484 - val_acc: 0.9816 - val_auc_roc: 0.9763\n",
      "Epoch 2/50\n"
     ]
    }
   ],
   "source": [
    "for k in kernels:\n",
    "    print (\"\\n********************k = {}********************\".format(k))\n",
    "    params[\"kernel_size\"] = k\n",
    "    result = run_5fold(x_train, y_train, x_test, \"loss\", get_CNN_model, **params)    \n",
    "    predictions_glove[k] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: 100\n",
      "loss: 0.04601445630171195 (+/- 0.0020066506932860306)\n",
      "acc: 0.9827829910552092 (+/- 0.0006525990545582435)\n",
      "roc_auc: 0.9865255406409961 (+/- 0.0009220619202532495)\n",
      "\n",
      "\n",
      "\n",
      "key: 300\n",
      "loss: 0.04493380877026916 (+/- 0.0008816936950137447)\n",
      "acc: 0.9830409674726788 (+/- 0.0002668991024840636)\n",
      "roc_auc: 0.9855141711177511 (+/- 0.0015221667521623432)\n",
      "\n",
      "\n",
      "\n",
      "key: 500\n",
      "loss: 0.046407338910416165 (+/- 0.00213438906316632)\n",
      "acc: 0.982989791722234 (+/- 0.00046590775908032697)\n",
      "roc_auc: 0.9857171781847021 (+/- 0.0011258855863419304)\n",
      "\n",
      "\n",
      "\n",
      "key: 5\n",
      "loss: 0.04748997002168405 (+/- 0.0021834484931673925)\n",
      "acc: 0.9826701848987014 (+/- 0.0005885760699276943)\n",
      "roc_auc: 0.9865483508044474 (+/- 0.0012091863265703145)\n",
      "\n",
      "\n",
      "\n",
      "key: 7\n",
      "loss: 0.04642165413081062 (+/- 0.0016689807822862668)\n",
      "acc: 0.9828007364559337 (+/- 0.0007606376298816129)\n",
      "roc_auc: 0.9848486750642831 (+/- 0.0013648982558387946)\n",
      "\n",
      "\n",
      "\n",
      "key: 9\n",
      "loss: 0.04623318673644024 (+/- 0.0012161839793471843)\n",
      "acc: 0.9829438316817724 (+/- 0.00017016266191228596)\n",
      "roc_auc: 0.9847446051121892 (+/- 0.0014313961638061227)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_results(predictions_glove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### result \n",
    "\n",
    "- 3:\n",
    "\n",
    "loss: 0.04493380877026916 (+/- 0.0008816936950137447)\n",
    "\n",
    "acc: 0.9830409674726788 (+/- 0.0002668991024840636)\n",
    "\n",
    "roc_auc: 0.9855141711177511 (+/- 0.0015221667521623432)\n",
    "\n",
    "- 5: \n",
    "\n",
    "loss: 0.04748997002168405 (+/- 0.0021834484931673925)\n",
    "\n",
    "acc: 0.9826701848987014 (+/- 0.0005885760699276943)\n",
    "\n",
    "roc_auc: 0.9865483508044474 (+/- 0.0012091863265703145)\n",
    "\n",
    "- 7: \n",
    "\n",
    "loss: 0.04642165413081062 (+/- 0.0016689807822862668)\n",
    "\n",
    "acc: 0.9828007364559337 (+/- 0.0007606376298816129)\n",
    "\n",
    "roc_auc: 0.9848486750642831 (+/- 0.0013648982558387946)\n",
    "\n",
    "- 9:\n",
    "\n",
    "loss: 0.04623318673644024 (+/- 0.0012161839793471843)\n",
    "\n",
    "acc: 0.9829438316817724 (+/- 0.00017016266191228596)\n",
    "\n",
    "roc_auc: 0.9847446051121892 (+/- 0.0014313961638061227)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_glove = opt_glove"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"kernel_size\": 3,\n",
    "    \"num_filter\": 300,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove,\n",
    "    \"embed_size\": embed_size_glove\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropouts = [0.1, 0.5] # 0.2 see previous result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "********************d = 0.1********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 57s - loss: 0.0522 - acc: 0.9813 - auc_roc: 0.9485 - val_loss: 0.0434 - val_acc: 0.9833 - val_auc_roc: 0.9802\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0374 - acc: 0.9856 - auc_roc: 0.9839 - val_loss: 0.0434 - val_acc: 0.9834 - val_auc_roc: 0.9857\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.043449460532963884\n",
      "val acc: 0.9834194758740336\n",
      "val auc roc: 0.9857266164546885\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0541 - acc: 0.9806 - auc_roc: 0.9427 - val_loss: 0.0440 - val_acc: 0.9831 - val_auc_roc: 0.9788\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0375 - acc: 0.9856 - auc_roc: 0.9829 - val_loss: 0.0475 - val_acc: 0.9820 - val_auc_roc: 0.9849\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.047493287397118646\n",
      "val acc: 0.9819566968121295\n",
      "val auc roc: 0.984937372867749\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0544 - acc: 0.9803 - auc_roc: 0.9388 - val_loss: 0.0429 - val_acc: 0.9835 - val_auc_roc: 0.9782\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0377 - acc: 0.9854 - auc_roc: 0.9826 - val_loss: 0.0442 - val_acc: 0.9836 - val_auc_roc: 0.9847\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.04421639789922827\n",
      "val acc: 0.9836435207053893\n",
      "val auc roc: 0.9846539370224889\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0530 - acc: 0.9809 - auc_roc: 0.9519 - val_loss: 0.0437 - val_acc: 0.9834 - val_auc_roc: 0.9806\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0375 - acc: 0.9855 - auc_roc: 0.9841 - val_loss: 0.0433 - val_acc: 0.9836 - val_auc_roc: 0.9857\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0289 - acc: 0.9888 - auc_roc: 0.9873 - val_loss: 0.0477 - val_acc: 0.9826 - val_auc_roc: 0.9883\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04772261597210059\n",
      "val acc: 0.9826356017620199\n",
      "val auc roc: 0.9882922162423645\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0535 - acc: 0.9806 - auc_roc: 0.9479 - val_loss: 0.0444 - val_acc: 0.9831 - val_auc_roc: 0.9795\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0373 - acc: 0.9856 - auc_roc: 0.9836 - val_loss: 0.0433 - val_acc: 0.9836 - val_auc_roc: 0.9854\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0292 - acc: 0.9887 - auc_roc: 0.9871 - val_loss: 0.0482 - val_acc: 0.9826 - val_auc_roc: 0.9878\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04820766794076986\n",
      "val acc: 0.9826460452824448\n",
      "val auc roc: 0.9878022399150078\n",
      "\n",
      "\n",
      "********************d = 0.5********************\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0581 - acc: 0.9796 - auc_roc: 0.9334 - val_loss: 0.0530 - val_acc: 0.9801 - val_auc_roc: 0.9748\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0438 - acc: 0.9835 - auc_roc: 0.9791 - val_loss: 0.0444 - val_acc: 0.9828 - val_auc_roc: 0.9815\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0396 - acc: 0.9848 - auc_roc: 0.9832 - val_loss: 0.0455 - val_acc: 0.9826 - val_auc_roc: 0.9843\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04546414956231019\n",
      "val acc: 0.9825578159044365\n",
      "val auc roc: 0.9843015525383242\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0580 - acc: 0.9797 - auc_roc: 0.9393 - val_loss: 0.0507 - val_acc: 0.9809 - val_auc_roc: 0.9754\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0440 - acc: 0.9834 - auc_roc: 0.9795 - val_loss: 0.0452 - val_acc: 0.9830 - val_auc_roc: 0.9818\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0398 - acc: 0.9849 - auc_roc: 0.9833 - val_loss: 0.0478 - val_acc: 0.9818 - val_auc_roc: 0.9843\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04783983414779893\n",
      "val acc: 0.9818261370231852\n",
      "val auc roc: 0.9843119591289634\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0580 - acc: 0.9798 - auc_roc: 0.9400 - val_loss: 0.0443 - val_acc: 0.9832 - val_auc_roc: 0.9753\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0439 - acc: 0.9834 - auc_roc: 0.9798 - val_loss: 0.0451 - val_acc: 0.9829 - val_auc_roc: 0.9819\n",
      "Epoch 00002: early stopping\n",
      "val loss: 0.04507317154280089\n",
      "val acc: 0.982933277540535\n",
      "val auc roc: 0.9818723954681975\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0594 - acc: 0.9790 - auc_roc: 0.9343 - val_loss: 0.0479 - val_acc: 0.9820 - val_auc_roc: 0.9746\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0444 - acc: 0.9832 - auc_roc: 0.9790 - val_loss: 0.0436 - val_acc: 0.9835 - val_auc_roc: 0.9813\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0399 - acc: 0.9847 - auc_roc: 0.9829 - val_loss: 0.0435 - val_acc: 0.9834 - val_auc_roc: 0.9840\n",
      "Epoch 4/50\n",
      " - 56s - loss: 0.0362 - acc: 0.9860 - auc_roc: 0.9850 - val_loss: 0.0446 - val_acc: 0.9833 - val_auc_roc: 0.9857\n",
      "Epoch 00004: early stopping\n",
      "val loss: 0.044644964022292614\n",
      "val acc: 0.9833197336025971\n",
      "val auc roc: 0.9857064070891055\n",
      "\n",
      "Train on 127657 samples, validate on 31914 samples\n",
      "Epoch 1/50\n",
      " - 56s - loss: 0.0589 - acc: 0.9793 - auc_roc: 0.9319 - val_loss: 0.0461 - val_acc: 0.9823 - val_auc_roc: 0.9741\n",
      "Epoch 2/50\n",
      " - 56s - loss: 0.0439 - acc: 0.9834 - auc_roc: 0.9786 - val_loss: 0.0457 - val_acc: 0.9825 - val_auc_roc: 0.9811\n",
      "Epoch 3/50\n",
      " - 56s - loss: 0.0397 - acc: 0.9847 - auc_roc: 0.9829 - val_loss: 0.0481 - val_acc: 0.9815 - val_auc_roc: 0.9839\n",
      "Epoch 00003: early stopping\n",
      "val loss: 0.04812161794225659\n",
      "val acc: 0.9815075702369007\n",
      "val auc roc: 0.9838945404280619\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for d in dropouts:\n",
    "    print (\"\\n********************d = {}********************\".format(d))\n",
    "    params[\"dropout\"] = d\n",
    "    result = run_5fold(x_train, y_train, x_test, \"loss\", get_CNN_model, **params)    \n",
    "    predictions_glove[d] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key: 100\n",
      "loss: 0.04601445630171195 (+/- 0.0020066506932860306)\n",
      "acc: 0.9827829910552092 (+/- 0.0006525990545582435)\n",
      "roc_auc: 0.9865255406409961 (+/- 0.0009220619202532495)\n",
      "\n",
      "\n",
      "\n",
      "key: 300\n",
      "loss: 0.04493380877026916 (+/- 0.0008816936950137447)\n",
      "acc: 0.9830409674726788 (+/- 0.0002668991024840636)\n",
      "roc_auc: 0.9855141711177511 (+/- 0.0015221667521623432)\n",
      "\n",
      "\n",
      "\n",
      "key: 500\n",
      "loss: 0.046407338910416165 (+/- 0.00213438906316632)\n",
      "acc: 0.982989791722234 (+/- 0.00046590775908032697)\n",
      "roc_auc: 0.9857171781847021 (+/- 0.0011258855863419304)\n",
      "\n",
      "\n",
      "\n",
      "key: 5\n",
      "loss: 0.04748997002168405 (+/- 0.0021834484931673925)\n",
      "acc: 0.9826701848987014 (+/- 0.0005885760699276943)\n",
      "roc_auc: 0.9865483508044474 (+/- 0.0012091863265703145)\n",
      "\n",
      "\n",
      "\n",
      "key: 7\n",
      "loss: 0.04642165413081062 (+/- 0.0016689807822862668)\n",
      "acc: 0.9828007364559337 (+/- 0.0007606376298816129)\n",
      "roc_auc: 0.9848486750642831 (+/- 0.0013648982558387946)\n",
      "\n",
      "\n",
      "\n",
      "key: 9\n",
      "loss: 0.04623318673644024 (+/- 0.0012161839793471843)\n",
      "acc: 0.9829438316817724 (+/- 0.00017016266191228596)\n",
      "roc_auc: 0.9847446051121892 (+/- 0.0014313961638061227)\n",
      "\n",
      "\n",
      "\n",
      "key: 0.1\n",
      "loss: 0.046217885948436246 (+/- 0.0019758657529593882)\n",
      "acc: 0.9828602680872034 (+/- 0.0006064660632937864)\n",
      "roc_auc: 0.9862824765004596 (+/- 0.0014912518010662232)\n",
      "\n",
      "\n",
      "\n",
      "key: 0.5\n",
      "loss: 0.04622874744349184 (+/- 0.0014564954240449707)\n",
      "acc: 0.982428906861531 (+/- 0.0006747995642684885)\n",
      "roc_auc: 0.9840173709305304 (+/- 0.0012358310243868592)\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_results(predictions_glove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### result\n",
    "- 0.1:\n",
    "\n",
    "loss: 0.046217885948436246 (+/- 0.0019758657529593882)\n",
    "\n",
    "acc: 0.9828602680872034 (+/- 0.0006064660632937864)\n",
    "\n",
    "roc_auc: 0.9862824765004596 (+/- 0.0014912518010662232)\n",
    "\n",
    "- 0.2:\n",
    "\n",
    "loss: 0.04493380877026916 (+/- 0.0008816936950137447)\n",
    "\n",
    "acc: 0.9830409674726788 (+/- 0.0002668991024840636)\n",
    "\n",
    "roc_auc: 0.9855141711177511 (+/- 0.0015221667521623432)\n",
    "\n",
    "\n",
    "- 0.5:\n",
    "\n",
    "loss: 0.04622874744349184 (+/- 0.0014564954240449707)\n",
    "\n",
    "acc: 0.982428906861531 (+/- 0.0006747995642684885)\n",
    "\n",
    "roc_auc: 0.9840173709305304 (+/- 0.0012358310243868592)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_glove = opt_glove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_prediction(opt_glove, submission, \"output/CNN_glove.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### params for glove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"maxlen\": maxlen,\n",
    "    \"max_features\": max_features,\n",
    "    \"kernel_size\": 3,\n",
    "    \"num_filter\": 300,\n",
    "    \"dropout\": 0.2,\n",
    "    \"reg\": 0.0,\n",
    "    \"embedding_matrix\": glove,\n",
    "    \"embed_size\": embed_size_glove\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}