QA/memn2n/n2nmn_base.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 구현하려는 것\n",
    "# https://github.com/vinhkhuc/MemN2N-babi-python\n",
    "# https://github.com/nmhkahn/MemN2N-pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "ctx = mx.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "data_dir = glob.glob('/home/hyungjunkim/Dropbox/repo/dl/MemN2N/scripts/' + '/data/')[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!cd /home/hyungjunkim/Dropbox/repo/dl/MemN2N/scripts/data && \\  \n",
    "wget https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz && \\  \n",
    "tar xvf babi_tasks_1-20_v1-2.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datautils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![task01](imgs/task_01.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![task02](imgs/task_02.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "task_id = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.5/re.py:203: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    }
   ],
   "source": [
    "train_data = bAbIDataset(data_dir + 'tasks_1-20_v1-2/en/', task_id=task_id, train=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.5/re.py:203: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    }
   ],
   "source": [
    "test_data = bAbIDataset(data_dir + 'tasks_1-20_v1-2/en/', task_id=task_id, train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_re = {value:key for key, value in train_data.word_idx.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 'back',\n",
       " 10: 'kitchen',\n",
       " 11: 'mary',\n",
       " 12: 'moved',\n",
       " 13: 'office',\n",
       " 14: 'sandra',\n",
       " 15: 'the',\n",
       " 16: 'to',\n",
       " 17: 'travelled',\n",
       " 18: 'went',\n",
       " 19: 'where',\n",
       " 2: 'bathroom',\n",
       " 3: 'bedroom',\n",
       " 4: 'daniel',\n",
       " 5: 'garden',\n",
       " 6: 'hallway',\n",
       " 7: 'is',\n",
       " 8: 'john',\n",
       " 9: 'journeyed',\n",
       " 'time1': 'time1',\n",
       " 'time10': 'time10',\n",
       " 'time2': 'time2',\n",
       " 'time3': 'time3',\n",
       " 'time4': 'time4',\n",
       " 'time5': 'time5',\n",
       " 'time6': 'time6',\n",
       " 'time7': 'time7',\n",
       " 'time8': 'time8',\n",
       " 'time9': 'time9'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_re "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11 12 16 15  2  0 21]\n",
      "mary moved to the bathroom .\n",
      "[ 8 18 16 15  6  0 20]\n",
      "john went to the hallway .\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "where is mary\n",
      "bathroom\n",
      "\n",
      "\n",
      "[11 12 16 15  2  0 23]\n",
      "mary moved to the bathroom .\n",
      "[ 8 18 16 15  6  0 22]\n",
      "john went to the hallway .\n",
      "[ 4 18  1 16 15  6 21]\n",
      "daniel went back to the hallway .\n",
      "[14 12 16 15  5  0 20]\n",
      "sandra moved to the garden .\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "where is daniel\n",
      "hallway\n",
      "\n",
      "\n",
      "[11 12 16 15  2  0 25]\n",
      "mary moved to the bathroom .\n",
      "[ 8 18 16 15  6  0 24]\n",
      "john went to the hallway .\n",
      "[ 4 18  1 16 15  6 23]\n",
      "daniel went back to the hallway .\n",
      "[14 12 16 15  5  0 22]\n",
      "sandra moved to the garden .\n",
      "[ 8 12 16 15 13  0 21]\n",
      "john moved to the office .\n",
      "[14  9 16 15  2  0 20]\n",
      "sandra journeyed to the bathroom .\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "[0 0 0 0 0 0 0]\n",
      "\n",
      "where is daniel\n",
      "hallway\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for ii, (x,y,z) in enumerate(train_data):\n",
    "    for idx in x:\n",
    "        print (idx)\n",
    "        print (\" \".join([word_re.get(int(i), '.') for i in idx if int(i) != 0]))\n",
    "    \n",
    "    print (\" \".join([word_re.get(int(i), '?') for i in y if int(i) != 0]))\n",
    "    print (word_re.get(int(z), '?'))\n",
    "    \n",
    "    print ('\\n')\n",
    "    if ii == 2:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs_train, queries_train, answers_train = train_data.data_story, train_data.data_query, train_data.data_answer\n",
    "inputs_test, queries_test, answers_test = test_data.data_story, test_data.data_query, test_data.data_answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def position_encoding(sentences, emb_dim):\n",
    "    s_tmp = sentences.asnumpy()\n",
    "    n_row,n_col = np.shape(s_tmp)\n",
    "    for i in range(n_row):\n",
    "        for j in range(n_col):\n",
    "            s_tmp[i,j] = (1-j/n_col) - (n_row/emb_dim)*(1-2*j/n_col)    \n",
    "    return mx.nd.array(s_tmp, ctx).expand_dims(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End-To-End Memory Network (N2NMN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "i = 문장   \n",
    "j = 단어"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inputs: $x_{i}$: 문장(context)  \n",
    "$q$: 질의  \n",
    "$target$: 답변(a word or words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![img0](imgs/n2nmn_fig0.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 메모리 벡터: 임베딩된 워드의 합을 문장으로 표기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$m_{i}={\\sum}_{j}Ax_{i,j}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Attention $P_{i}$: 질의(q)와 가장 관련된 Senence(context) 확률로 나타냄"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$P_i=Softmax(uT {\\cdot} mi)$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$o={\\sum}_{i}p_{i}c_{i}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$a=Softmax(W(o+u))$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "internal state $u={\\sum}_{j}Bq_{j}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 학습: A, C, B, W"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### multi-layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "adjacent: output embedding(o1 + u1)이 새로운 input embedding이 됨   \n",
    "Layer-wise (RNN-like): 모든 레이어가 A와 C를 공유함.  $uk^{+1} = Hu^{k}+o^{k}$, linear mapping H"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 성능 올리기: 1. Senetence Representation, 2. Terporal Encoding, 3. Lerning time invariance by injection random noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 10, 7)\n",
      "(1000, 7)\n",
      "(1000,)\n"
     ]
    }
   ],
   "source": [
    "print (np.shape(inputs_train))\n",
    "print (np.shape(queries_train))\n",
    "print (np.shape(answers_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import autograd, gluon, nd, init\n",
    "from mxnet.gluon import Block, nn\n",
    "\n",
    "\n",
    "class MemN2N(Block):\n",
    "    def __init__(self, config, **kwargs):\n",
    "        super(MemN2N, self).__init__(**kwargs)\n",
    "\n",
    "        ## 고정\n",
    "        self.vocab_size = len(word_re)\n",
    "        \n",
    "        ## \n",
    "        self.init_std = 0.015\n",
    "        self.nhop = 3\n",
    "        self.emb_dim = 50\n",
    "\n",
    "        with self.name_scope():\n",
    "            \n",
    "            # Embedding A (메모리 벡터)\n",
    "            self.A = nn.Embedding(input_dim=self.vocab_size, \n",
    "                                  output_dim=self.emb_dim, \n",
    "                                  weight_initializer=init.Normal(self.init_std))\n",
    "             \n",
    "            # Embedding C (컨텍스트 벡터)\n",
    "            self.C = nn.Embedding(input_dim=self.vocab_size,\n",
    "                                  output_dim=self.emb_dim, \n",
    "                                  weight_initializer=init.Normal(self.init_std))\n",
    "            \n",
    "            # Embedding Q (쿼리 벡터)\n",
    "            self.B = nn.Embedding(input_dim=self.vocab_size, \n",
    "                                  output_dim=self.emb_dim, \n",
    "                                  weight_initializer=init.Normal(self.init_std))\n",
    "\n",
    "            # Final Predict \n",
    "            self.W = nn.Dense(self.vocab_size, weight_initializer=init.Normal(self.init_std))\n",
    "            \n",
    "    def forward(self, sentences, question):\n",
    "\n",
    "        hid = []\n",
    "        hid.append(question)\n",
    "        \n",
    "        # 16, 50, 7 , 19 > 16, 50, 19\n",
    "        m_i = self.A(sentences) #* position_encoding(sentences, self.emb_dim)\n",
    "        m_i = m_i.sum(axis=2)\n",
    "        \n",
    "        # 16, 50, 7 , 19 > 16, 50, 19\n",
    "        c_i = self.C(sentences)\n",
    "        c_i = c_i.sum(axis=2)\n",
    "            \n",
    "        for n in range(self.nhop):\n",
    "            \n",
    "            if n == 0:\n",
    "                #  16, 7, 50 > 16, 50\n",
    "                u = self.B(hid[-1])# * position_encoding(hid[-1], self.emb_dim)\n",
    "                u = u.sum(axis=1)\n",
    "            else:\n",
    "                u = hid[-1]\n",
    "\n",
    "            # [16, 1, 50] [16, 50, 10]  > [16, 1, 50]\n",
    "            m_out = nd.batch_dot(u.expand_dims(1), m_i.swapaxes(1,2))\n",
    "            P = nd.softmax(m_out, axis=2)\n",
    "            \n",
    "            #Cout = P * c_i\n",
    "            # [16, 1, 10] [16, 10, 50] > [16, 1, 50]\n",
    "            Cout = nd.batch_dot(P,  c_i)\n",
    "            # [16, 1, 50] > [16, 50]\n",
    "            o = Cout.reshape((0,-1))#Cout.sum(axis=(1))\n",
    "            \n",
    "            ## O + U [16, 50]\n",
    "            Dout = o + u\n",
    "            hid.append(Dout)\n",
    "           \n",
    "        z = self.W(hid[-1])\n",
    "        return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, trainer, x_input, x_query, x_answers, batch_size):\n",
    "    softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "    max_grad_norm=40\n",
    "    N = int(math.ceil(len(x_input) / batch_size))\n",
    "    cost = 0.0\n",
    "\n",
    "    for idx in range(N):\n",
    "        with autograd.record():\n",
    "            out = model(x_input, x_query)\n",
    "            loss = softmax_cross_entropy(out, x_answers)\n",
    "            loss.backward()\n",
    "            \n",
    "        grads = [i.grad() for i in model.collect_params().values()]\n",
    "        gluon.utils.clip_global_norm(grads, max_grad_norm)\n",
    "        trainer.step(batch_size)\n",
    "        cost += nd.sum(loss).asscalar()\n",
    "\n",
    "    return cost/N/batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MemN2N(None)\n",
    "model.collect_params().initialize(mx.init.Xavier(),ctx=ctx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MemN2N(\n",
      "  (C): Embedding(29 -> 50, float32)\n",
      "  (W): Dense(None -> 29, linear)\n",
      "  (A): Embedding(29 -> 50, float32)\n",
      "  (B): Embedding(29 -> 50, float32)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print (model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "trainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': 0.01})\n",
    "log_loss = []\n",
    "log_perp = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "epochs  = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epoch': 0, 'perplexity': 6.115564203702698, 'learning_rate': 0.01}\n",
      "{'epoch': 1, 'perplexity': 5.520157808431111, 'learning_rate': 0.01}\n",
      "{'epoch': 2, 'perplexity': 5.631711224203392, 'learning_rate': 0.01}\n",
      "update learning rate from 0.010 to 0.007\n",
      "{'epoch': 3, 'perplexity': 3.2840659214117527, 'learning_rate': 0.006666666666666667}\n",
      "{'epoch': 4, 'perplexity': 1.6909363733537708, 'learning_rate': 0.006666666666666667}\n",
      "{'epoch': 5, 'perplexity': 1.1349282690956812, 'learning_rate': 0.006666666666666667}\n",
      "{'epoch': 6, 'perplexity': 1.0153575650558406, 'learning_rate': 0.006666666666666667}\n",
      "{'epoch': 7, 'perplexity': 1.0066820414378812, 'learning_rate': 0.006666666666666667}\n",
      "{'epoch': 8, 'perplexity': 1.0028816768086037, 'learning_rate': 0.006666666666666667}\n",
      "{'epoch': 9, 'perplexity': 1.0014552920344273, 'learning_rate': 0.006666666666666667}\n",
      "{'epoch': 10, 'perplexity': 1.0028680910778285, 'learning_rate': 0.006666666666666667}\n",
      "update learning rate from 0.007 to 0.004\n",
      "{'epoch': 11, 'perplexity': 1.0025687447850256, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 12, 'perplexity': 1.0008211286397468, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 13, 'perplexity': 1.0006501275987572, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 14, 'perplexity': 1.0005616132013986, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 15, 'perplexity': 1.0004966571090697, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 16, 'perplexity': 1.0004448375126747, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 17, 'perplexity': 1.0004021224822828, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 18, 'perplexity': 1.0003661068368364, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 19, 'perplexity': 1.0003353085523814, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 20, 'perplexity': 1.0003085286302378, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 21, 'perplexity': 1.000285081884694, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 22, 'perplexity': 1.0002643910908005, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 23, 'perplexity': 1.0002459796178813, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 24, 'perplexity': 1.0002295756079769, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 25, 'perplexity': 1.0002148364675898, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 26, 'perplexity': 1.0002014977342906, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 27, 'perplexity': 1.0001894216502993, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 28, 'perplexity': 1.0001783884404074, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 29, 'perplexity': 1.0001683646387278, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 30, 'perplexity': 1.0001591937860186, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 31, 'perplexity': 1.0001506672984328, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 32, 'perplexity': 1.0001428448163117, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 33, 'perplexity': 1.000135614549266, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 34, 'perplexity': 1.0001289318479731, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 35, 'perplexity': 1.0001227147495646, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 36, 'perplexity': 1.0001169632671885, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 37, 'perplexity': 1.0001116066189475, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 38, 'perplexity': 1.0001066112891406, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 39, 'perplexity': 1.0001019549256098, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 40, 'perplexity': 1.0000975965632575, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 41, 'perplexity': 1.0000935063930778, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 42, 'perplexity': 1.0000896285419696, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 43, 'perplexity': 1.0000859965525115, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 44, 'perplexity': 1.0000825731514922, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 45, 'perplexity': 1.000079362084185, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 46, 'perplexity': 1.0000763037393507, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 47, 'perplexity': 1.0000734167506828, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 48, 'perplexity': 1.0000707122788062, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 49, 'perplexity': 1.0000681419192705, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 50, 'perplexity': 1.000065683310555, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 51, 'perplexity': 1.0000633774253491, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 52, 'perplexity': 1.0000611684016512, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 53, 'perplexity': 1.0000590599461385, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 54, 'perplexity': 1.0000570744221076, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 55, 'perplexity': 1.0000552118288253, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 56, 'perplexity': 1.0000533715810322, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 57, 'perplexity': 1.0000516244627722, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 58, 'perplexity': 1.0000499369442692, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 59, 'perplexity': 1.0000483760873269, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 60, 'perplexity': 1.000046818947371, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 61, 'perplexity': 1.0000453884681746, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 62, 'perplexity': 1.0000440026891975, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 63, 'perplexity': 1.0000426467084575, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 64, 'perplexity': 1.0000413726856918, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 65, 'perplexity': 1.0000401694406296, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 66, 'perplexity': 1.0000390034440938, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 67, 'perplexity': 1.0000378709777995, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 68, 'perplexity': 1.0000368204649124, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 69, 'perplexity': 1.0000357475969495, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 70, 'perplexity': 1.0000347529603115, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 71, 'perplexity': 1.0000337806735213, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 72, 'perplexity': 1.0000328642688658, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 73, 'perplexity': 1.000031977661049, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 74, 'perplexity': 1.000031132033486, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 75, 'perplexity': 1.0000302752286099, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 76, 'perplexity': 1.0000294929288305, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 77, 'perplexity': 1.0000287143532411, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 78, 'perplexity': 1.0000279953809317, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 79, 'perplexity': 1.0000272615075714, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 80, 'perplexity': 1.000026557436044, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 81, 'perplexity': 1.0000258868934926, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 82, 'perplexity': 1.0000252312492899, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 83, 'perplexity': 1.0000245979578082, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 84, 'perplexity': 1.0000239795664267, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 85, 'perplexity': 1.000023398427383, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 86, 'perplexity': 1.0000228284629835, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 87, 'perplexity': 1.0000222547735358, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 88, 'perplexity': 1.0000217146127575, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 89, 'perplexity': 1.0000212079787751, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 90, 'perplexity': 1.0000206938924954, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 91, 'perplexity': 1.0000202245108667, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 92, 'perplexity': 1.0000197588530033, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 93, 'perplexity': 1.0000192931953567, 'learning_rate': 0.0044444444444444444}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epoch': 94, 'perplexity': 1.0000188573408106, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 95, 'perplexity': 1.0000183991343015, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 96, 'perplexity': 1.0000179781797554, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 97, 'perplexity': 1.000017590751769, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 98, 'perplexity': 1.0000171586215003, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 99, 'perplexity': 1.0000168047220073, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 100, 'perplexity': 1.00001640239307, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 101, 'perplexity': 1.0000160112403444, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 102, 'perplexity': 1.000015683416879, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 103, 'perplexity': 1.0000152885390874, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 104, 'perplexity': 1.0000149383646926, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 105, 'perplexity': 1.0000145956402, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 106, 'perplexity': 1.0000142864430066, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 107, 'perplexity': 1.000013966069882, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 108, 'perplexity': 1.000013671774248, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 109, 'perplexity': 1.0000133551257506, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 110, 'perplexity': 1.000013068280065, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 111, 'perplexity': 1.0000128149615939, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 112, 'perplexity': 1.0000125206653891, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 113, 'perplexity': 1.0000122598963848, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 114, 'perplexity': 1.0000120140287885, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 115, 'perplexity': 1.000011760709675, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 116, 'perplexity': 1.000011526018201, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 117, 'perplexity': 1.0000112987765375, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 118, 'perplexity': 1.0000110640842625, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 119, 'perplexity': 1.000010825666712, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 120, 'perplexity': 1.000010605875867, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 121, 'perplexity': 1.0000104233374516, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 122, 'perplexity': 1.0000102035466953, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 123, 'perplexity': 1.0000099986563877, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 124, 'perplexity': 1.000009797491449, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 125, 'perplexity': 1.000009611227855, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 126, 'perplexity': 1.0000094100629942, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 127, 'perplexity': 1.0000092349754464, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 128, 'perplexity': 1.0000090524363716, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 129, 'perplexity': 1.0000088438209767, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 130, 'perplexity': 1.0000086761832638, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 131, 'perplexity': 1.0000085122718105, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 132, 'perplexity': 1.000008352084796, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 133, 'perplexity': 1.0000082067990903, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 134, 'perplexity': 1.0000080466121248, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 135, 'perplexity': 1.0000078715239062, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 136, 'perplexity': 1.0000077336889086, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 137, 'perplexity': 1.0000075958534753, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 138, 'perplexity': 1.0000074580180611, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 139, 'perplexity': 1.0000073052818506, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 140, 'perplexity': 1.0000071748966555, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 141, 'perplexity': 1.0000070407866157, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 142, 'perplexity': 1.000006902951278, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 143, 'perplexity': 1.0000067874678524, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 144, 'perplexity': 1.0000066459072345, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 145, 'perplexity': 1.0000065080719513, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 146, 'perplexity': 1.0000063925885714, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 147, 'perplexity': 1.0000062808300636, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 148, 'perplexity': 1.0000061467196892, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 149, 'perplexity': 1.0000060461371474, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 150, 'perplexity': 1.0000059194778836, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 151, 'perplexity': 1.0000058263459888, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 152, 'perplexity': 1.0000057034116097, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 153, 'perplexity': 1.0000055916531791, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 154, 'perplexity': 1.0000054873453825, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 155, 'perplexity': 1.0000053718616657, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 156, 'perplexity': 1.0000052526526524, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 157, 'perplexity': 1.000005170696748, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 158, 'perplexity': 1.0000050924657051, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 159, 'perplexity': 1.000004976982034, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 160, 'perplexity': 1.0000048726743014, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 161, 'perplexity': 1.00000479816859, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 162, 'perplexity': 1.0000047050367995, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 163, 'perplexity': 1.000004619355178, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 164, 'perplexity': 1.0000045150470278, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 165, 'perplexity': 1.0000044517172637, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 166, 'perplexity': 1.0000043585855054, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 167, 'perplexity': 1.000004295255751, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 168, 'perplexity': 1.0000042170247765, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 169, 'perplexity': 1.0000041313431969, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 170, 'perplexity': 1.0000040568375406, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 171, 'perplexity': 1.0000039972331105, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 172, 'perplexity': 1.0000039264527691, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 173, 'perplexity': 1.0000038705731968, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 174, 'perplexity': 1.000003773715961, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 175, 'perplexity': 1.0000037103860162, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 176, 'perplexity': 1.0000036358803914, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 177, 'perplexity': 1.0000035800010627, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 178, 'perplexity': 1.0000035092205235, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 179, 'perplexity': 1.0000034421655197, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 180, 'perplexity': 1.0000033825611263, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 181, 'perplexity': 1.0000033304068872, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 182, 'perplexity': 1.0000032894287856, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 183, 'perplexity': 1.0000032037472852, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 184, 'perplexity': 1.0000031515932828, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 185, 'perplexity': 1.0000031031645849, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 186, 'perplexity': 1.0000030472852859, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 187, 'perplexity': 1.0000029951315188, 'learning_rate': 0.0044444444444444444}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epoch': 188, 'perplexity': 1.0000029392519987, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 189, 'perplexity': 1.000002894548612, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 190, 'perplexity': 1.0000028461199264, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 191, 'perplexity': 1.0000027827900406, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 192, 'perplexity': 1.0000027343613604, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 193, 'perplexity': 1.000002704558957, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 194, 'perplexity': 1.0000026449543802, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 195, 'perplexity': 1.0000025965257067, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 196, 'perplexity': 1.000002562998235, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 197, 'perplexity': 1.0000025145695655, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 198, 'perplexity': 1.0000024698661978, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 199, 'perplexity': 1.000002425162605, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 200, 'perplexity': 1.0000023916351388, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 201, 'perplexity': 1.0000023581074464, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 202, 'perplexity': 1.0000023134040859, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 203, 'perplexity': 1.0000022649754285, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 204, 'perplexity': 1.000002235173039, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 205, 'perplexity': 1.0000021867443856, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 206, 'perplexity': 1.0000021532167, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 207, 'perplexity': 1.0000021271398396, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 208, 'perplexity': 1.0000020824362619, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 209, 'perplexity': 1.0000020451835094, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 210, 'perplexity': 1.0000020004799355, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 211, 'perplexity': 1.0000019744030788, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 212, 'perplexity': 1.000001944600698, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 213, 'perplexity': 1.0000019036225396, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 214, 'perplexity': 1.0000018700949773, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 215, 'perplexity': 1.0000018551937884, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 216, 'perplexity': 1.0000018253914111, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 217, 'perplexity': 1.0000017993144454, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 218, 'perplexity': 1.000001758336293, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 219, 'perplexity': 1.0000017248087356, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 220, 'perplexity': 1.0000017061823658, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 221, 'perplexity': 1.0000016726546965, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 222, 'perplexity': 1.0000016428524385, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 223, 'perplexity': 1.0000016242259564, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 224, 'perplexity': 1.0000015869731074, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 225, 'perplexity': 1.0000015534455557, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 226, 'perplexity': 1.0000015348190754, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 227, 'perplexity': 1.0000015012915255, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 228, 'perplexity': 1.0000014752144546, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 229, 'perplexity': 1.0000014528627938, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 230, 'perplexity': 1.0000014379616111, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 231, 'perplexity': 1.000001404433951, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 232, 'perplexity': 1.000001385807587, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 233, 'perplexity': 1.000001359730519, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 234, 'perplexity': 1.000001337378861, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 235, 'perplexity': 1.0000013113017943, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 236, 'perplexity': 1.0000012889500234, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 237, 'perplexity': 1.0000012703236618, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 238, 'perplexity': 1.0000012442465969, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 239, 'perplexity': 1.0000012181695328, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 240, 'perplexity': 1.0000012069937618, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 241, 'perplexity': 1.000001177191404, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 242, 'perplexity': 1.0000011511144553, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 243, 'perplexity': 1.000001128762688, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 244, 'perplexity': 1.0000011101362156, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 245, 'perplexity': 1.00000107660868, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 246, 'perplexity': 1.0000010728833857, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 247, 'perplexity': 1.0000010542569144, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 248, 'perplexity': 1.0000010393557377, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 249, 'perplexity': 1.000001024454561, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 250, 'perplexity': 1.0000010132787924, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 251, 'perplexity': 1.0000010021029102, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 252, 'perplexity': 1.0000009872017344, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 253, 'perplexity': 1.0000009760258524, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 254, 'perplexity': 1.0000009573993829, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 255, 'perplexity': 1.000000916421208, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 256, 'perplexity': 1.0000009089706774, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 257, 'perplexity': 1.0000008866189156, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 258, 'perplexity': 1.0000008717177413, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 259, 'perplexity': 1.0000008568165673, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 260, 'perplexity': 1.0000008419154502, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 261, 'perplexity': 1.0000008270142766, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 262, 'perplexity': 1.0000008083878669, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 263, 'perplexity': 1.0000007934866937, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 264, 'perplexity': 1.0000007748602275, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 265, 'perplexity': 1.000000763684348, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 266, 'perplexity': 1.0000007562337618, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 267, 'perplexity': 1.0000007487831757, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 268, 'perplexity': 1.0000007450579393, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 269, 'perplexity': 1.0000007376073532, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 270, 'perplexity': 1.0000007189808882, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 271, 'perplexity': 1.0000007040797732, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 272, 'perplexity': 1.0000006966291874, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 273, 'perplexity': 1.000000681728016, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 274, 'perplexity': 1.000000678002723, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 275, 'perplexity': 1.0000006519257305, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 276, 'perplexity': 1.0000006370245598, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 277, 'perplexity': 1.0000006295739745, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 278, 'perplexity': 1.0000006258486818, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 279, 'perplexity': 1.000000614672804, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 280, 'perplexity': 1.0000006072222187, 'learning_rate': 0.0044444444444444444}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epoch': 281, 'perplexity': 1.000000596046398, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 282, 'perplexity': 1.000000588595813, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 283, 'perplexity': 1.000000581145228, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 284, 'perplexity': 1.0000005774199356, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 285, 'perplexity': 1.0000005662440583, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 286, 'perplexity': 1.0000005513428887, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 287, 'perplexity': 1.000000543892304, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 288, 'perplexity': 1.0000005252658426, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 289, 'perplexity': 1.0000005140900228, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 290, 'perplexity': 1.0000005066394384, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 291, 'perplexity': 1.000000495463562, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 292, 'perplexity': 1.0000004805623934, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 293, 'perplexity': 1.0000004768371014, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 294, 'perplexity': 1.0000004731118093, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 295, 'perplexity': 1.0000004656612251, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 296, 'perplexity': 1.0000004582106412, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 297, 'perplexity': 1.0000004507600857, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 298, 'perplexity': 1.0000004470347936, 'learning_rate': 0.0044444444444444444}\n",
      "{'epoch': 299, 'perplexity': 1.0000004358589178, 'learning_rate': 0.0044444444444444444}\n"
     ]
    }
   ],
   "source": [
    "epoch_log_loss = []\n",
    "import math\n",
    "for epoch in range(epochs):\n",
    "    dataiter = mx.io.NDArrayIter([inputs_train, queries_train], answers_train, batch_size, shuffle=False, last_batch_handle='discard')\n",
    "    \n",
    "    for batch in dataiter:\n",
    "        log_loss = []\n",
    "        train_loss = train(model, trainer, \n",
    "                           batch.data[0].as_in_context(ctx), \n",
    "                           batch.data[1].as_in_context(ctx),\n",
    "                           batch.label[0].as_in_context(ctx),\n",
    "                           batch_size)\n",
    "\n",
    "        # Logging\n",
    "        log_loss.append([train_loss])#, test_loss])\n",
    "    \n",
    "    epoch_log_loss.append([np.mean(log_loss)])\n",
    "    \n",
    "    state = { 'epoch': epoch, 'learning_rate': trainer.learning_rate, 'perplexity': math.exp(epoch_log_loss[epoch][0])}\n",
    "    print(state)\n",
    "    \n",
    "    lr_decay = 1.5\n",
    "    if (len(epoch_log_loss) > 1) and (epoch_log_loss[epoch][0] > epoch_log_loss[epoch-1][0] * 0.9999):\n",
    "        print ('update learning rate from %.3f to %.3f' % (trainer.learning_rate, trainer.learning_rate/lr_decay))\n",
    "        trainer.set_learning_rate(trainer.learning_rate / lr_decay)\n",
    "    if trainer.learning_rate < 1e-5: \n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "memn2n0_ (\n",
       "  Parameter memn2n0_embedding0_weight (shape=(29, 50), dtype=<class 'numpy.float32'>)\n",
       "  Parameter memn2n0_embedding1_weight (shape=(29, 50), dtype=<class 'numpy.float32'>)\n",
       "  Parameter memn2n0_embedding2_weight (shape=(29, 50), dtype=<class 'numpy.float32'>)\n",
       "  Parameter memn2n0_dense0_weight (shape=(29, 50), dtype=<class 'numpy.float32'>)\n",
       "  Parameter memn2n0_dense0_bias (shape=(29,), dtype=<class 'numpy.float32'>)\n",
       ")"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.collect_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_ = model(mx.nd.array(inputs_test, mx.cpu()), mx.nd.array(queries_test, mx.cpu()))\n",
    "pred_ = pred_.asnumpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{2, 3, 5, 6, 10, 13}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set([np.argmax(i) for i in pred_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{2, 3, 5, 6, 10, 13}\n",
      "{2, 3, 5, 6, 10, 13}\n"
     ]
    }
   ],
   "source": [
    "print (set(np.argmax(i) for i in pred_))\n",
    "print (set(i for i in answers_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 2, 10, 6, 10, 6, 5, 6, 13, 13]\n",
      "[6, 2, 10, 6, 10, 6, 5, 6, 13, 13]\n"
     ]
    }
   ],
   "source": [
    "print ([np.argmax(i) for i in pred_[0:10]])\n",
    "print ([i for i in answers_test[0:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.996"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum([bool(np.argmax(i) == answers_test[idx:idx+1]) for idx, i in enumerate(pred_)])/len(pred_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.16666666666666666\n"
     ]
    }
   ],
   "source": [
    "## base rate\n",
    "print (1/len(set(i for i in answers_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "john travelled to the hallway .\n",
      "mary journeyed to the bathroom .\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "where is john\n",
      "hallway\n"
     ]
    }
   ],
   "source": [
    "for ido in test_data.data_story[idx:idx+1][0]:\n",
    "    print (\" \".join([word_re.get(int(i), '.') for i in ido if int(i) != 0]))\n",
    "    \n",
    "print (\" \".join([word_re.get(int(i), '.') for i in test_data.data_query[idx:idx+1][0] if int(i) != 0]))\n",
    "print (word_re.get(int(test_data.data_answer[idx:idx+1][0]), '.'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hallway\n"
     ]
    }
   ],
   "source": [
    "print (word_re[ np.argmax(pred_[idx])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}