Day 04 Attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Attention to Translation",
      "provenance": [],
      "authorship_tag": "ABX9TyPmpeWFcbToLDUimi65X/yF",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lagom-QB/M12/blob/master/Day%2004%20Attention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fpGoIezBgdEI",
        "colab_type": "text"
      },
      "source": [
        "#Name Generation (RNN)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "475YtgUygbyl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ---------- Import things\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import os\n",
        "\n",
        "from IPython.display import clear_output\n",
        "from random import sample"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UZOd0MDsg0rB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "start_token = \" \"\n",
        "\n",
        "def read_names(path_to_file):\n",
        "    global start_token\n",
        "\n",
        "    with open(path_to_file) as f:\n",
        "        names = f.read()[:-1].split('\\n')\n",
        "        names = [start_token + line for line in names]\n",
        "        return names"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fIUWVvGZiUtj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165
        },
        "outputId": "a6583ec2-e0b9-48ca-80a0-c0b14ea81020"
      },
      "source": [
        "try:\n",
        "    names = read_names('../datasets/names_dataset/names')\n",
        "except FileNotFoundError:\n",
        "    !wget https://raw.githubusercontent.com/neychev/harbour_dlia2020/master/datasets/names_dataset/names -nc -O names\n",
        "    names = read_names('./names')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-18 18:39:10--  https://raw.githubusercontent.com/neychev/harbour_dlia2020/master/datasets/names_dataset/names\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 55868 (55K) [text/plain]\n",
            "Saving to: ‘names’\n",
            "\n",
            "\rnames                 0%[                    ]       0  --.-KB/s               \rnames               100%[===================>]  54.56K  --.-KB/s    in 0.02s   \n",
            "\n",
            "2020-06-18 18:39:10 (3.52 MB/s) - ‘names’ saved [55868/55868]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVZf9Ga-iZjm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165
        },
        "outputId": "c7aa697d-d219-4a82-de7c-636c7c555f58"
      },
      "source": [
        "try:\n",
        "    names_ru = read_names('../datasets/names_dataset/names_ru')\n",
        "except FileNotFoundError:\n",
        "    !wget https://raw.githubusercontent.com/neychev/harbour_dlia2020/master/datasets/names_dataset/names_ru -nc -O names_ru\n",
        "    names_ru = read_names('./names_ru')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-18 18:39:12--  https://raw.githubusercontent.com/neychev/harbour_dlia2020/master/datasets/names_dataset/names_ru\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 103242 (101K) [text/plain]\n",
            "Saving to: ‘names_ru’\n",
            "\n",
            "\rnames_ru              0%[                    ]       0  --.-KB/s               \rnames_ru            100%[===================>] 100.82K  --.-KB/s    in 0.02s   \n",
            "\n",
            "2020-06-18 18:39:12 (4.18 MB/s) - ‘names_ru’ saved [103242/103242]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0Hy-LmyidzC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "04699814-683e-47fc-a8ea-c497bd246cf5"
      },
      "source": [
        "print ('n samples = ',len(names_ru))\n",
        "for idx in np.arange(0, len(names), 1000):\n",
        "    print(names[idx], names_ru[idx])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "n samples =  7944\n",
            " Abagael  Абагаел\n",
            " Claresta  Слареста\n",
            " Glory  Глорй\n",
            " Liliane  Лилиане\n",
            " Prissie  Приссие\n",
            " Geeta  Геета\n",
            " Giovanne  Гиованне\n",
            " Piggy  Пиггй\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MhqTnhBFii9Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        },
        "outputId": "147c5d47-2206-4c0d-a139-da6b553055d9"
      },
      "source": [
        "MAX_LENGTH = max(map(len, names))\n",
        "print(\"max length =\", MAX_LENGTH)\n",
        "\n",
        "plt.title('Sequence length distribution')\n",
        "plt.hist(list(map(len, names)),bins=22, label='en');\n",
        "\n",
        "plt.title('Sequence length distribution')\n",
        "plt.hist(list(map(len, names_ru)),bins=22, alpha=0.5, label='ru');\n",
        "plt.legend()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "max length = 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f2e5ad2f9e8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAdPUlEQVR4nO3df5xWdZ338dc7QPG3KCMqgw4ZUsojMSejLUtvN1PjDut2/bHdAWahm9Zy5+P2FnM3NuWObXPVHhktKoG3hrqYK5uWElu5tmEORoghiYoxNMAI/ih/JfC5/zjfseM4w1y/Zq4Zzvv5eFyPOef7/Z7v+ZxrZj7X9/qec11HEYGZmRXD2+odgJmZ9R0nfTOzAnHSNzMrECd9M7MCcdI3MysQJ30zswJx0rddmqSQ9I467PdESa1VbD9T0i1p+TBJf5Q0qEaxfUfS39Uizi76PkHSmlr1Z7XnpF8Akj4o6b8kvSBpq6SfS3pvvePalfTmi0tE/C4i9o6I7T3EMFXSgyX0d2FEXFmL2Dofd0T8Z0SMrUXf1jsG1zsA612S9gV+APwNcAewG3AC8Fo947L6kDSopxcP27V5pL/rOxIgIhZGxPaIeCUi7o+IlR0NJH1G0mpJz0m6T9LhubqPSHo8vUv4lqSfSfpsqntjCiKtN6WR3+C0vp+kmyS1Sdog6aqOKYqOUamkb6T9Pi3ptFxfB0j6rqTfp/p/y9VNlLRC0vPpHcy7S3kiJO2e9vc7SZvSNMceqe5ESa2SLpG0OcV8Xm7bAyX9u6QXJT2cjuXBVPdAavbrNA1zdm67LvvrIrbR6bn9g6QlwPCdPK9TJT2V2j4t6VOS3gV8B3h/iuH51Ha+pDmS7pX0EnBSKruq0/4vl/SspHWSPpUr/2nH7zv/e+vuuDtPF0l6V+rjeUmPSfp4rm6+pOsl3ZOO5SFJR/T0e7TqOOnv+n4LbJe0QNJpkoblKyVNAi4HPgk0AP8JLEx1w4HvA1eQJaEngQ+Use/5wDbgHcCxwCnAZ3P17wPWpL6/DtwkSanu/wF7AkcDBwHXpJiOBeYBFwAHAv8CLJa0ewnxzCZ7ERyfYhoJ/H2u/mBgv1R+PnB97vm6HngptZmSHgBExIfS4jFpGub2Evrr7HvA8vRcXJnvP0/SXsA3gdMiYh/gL4AVEbEauBD4RYph/9xmfw3MAvYBupr+OTjtd2Ta71xJPU7R7OS4O2IdAvw7cD/Z7/ALwK2d+j4H+AdgGLA2xWm9KSL82MUfwLvIEnArWRJeDIxIdT8Ezs+1fRvwMnA4MBlYlqtT6uOzaX0mcEuuvgkIsmnDEWRTSHvk6s8FfpKWpwJrc3V7pm0PBg4BdgDDujiWOcCVncrWAB/u5tiDLMGLLGkfkat7P/B0Wj4ReAUYnKvfDEwABgGvA2NzdVcBD3beT2692/66iPGw9HvZK1f2vY7nttPzuhfwPPA/8s9t7jl9sFPZfODmLsquysXZed93AH+Xln/a8fvuah/dHHdrWj4B2Ai8LVe/EJiZi+PGXN3pwOP1/n/Z1R8e6RdARKyOiKkR0QiMAw4Frk3VhwPXpbffzwNbyRLkyNRufa6fyK/34HBgCNCW6/tfyEZ8HTbm+n45Le4NjAK2RsRz3fR7SUefqd9RKdadaSB7YVme2+5HqbzDlojYllt/OcXTQJZw88deyvPQXX+dHQo8FxEv5cqe6arD1OZsslF9W5oaeWcPcfQUa1f77un5LMWhwPqI2NGp75G59Y255e6eH6shJ/2CiYjHyUZY41LReuCCiNg/99gjIv4LaCNLqACkqZdRue5eIkukHQ7OLa8nG+kPz/W7b0QcXUKY64EDJO3fTd2sTvHuGRELe+jzWbKR99G57faLiFKSTDvZaLgxVzaqm7aVaAOGpambDod11zgi7ouIj5C9I3ocuKGjqrtNeth/V/v+fVre2e+4J78HRknK55nDgA1l9GE15qS/i5P0znQysTGtjyKbZlmWmnwHmCHp6FS/n6S/SnX3AEdL+mQ6ifhF3vxPvwL4kLLryPcDZnRUREQb2Vzu1ZL2lfQ2SUdI+nBPMadtfwh8W9IwSUMkdcwf3wBcKOl9yuwl6WOS9umhzx1p22skHZSOdaSkj5YQz3aycxszJe2ZRtaTOzXbBLy9p7666f8ZoAX4B0m7Sfog8N+7aitphKRJKUm/BvyRbCqsI4ZGSbtVEEbHvk8AJgL/mspXAJ9Mx/0OsnMTeTs77ofIRu+Xpt/hiem4bqsgPqsRJ/1d3x/ITpg+lK7eWAasAi4BiIi7gH8EbpP0Yqo7LdU9C/wV2QnQLcAY4OcdHUfEEuB2YCXZScgfdNr3ZLJLRH8DPAcsIhudluLTZPPoj5PNhU9P+2wBPgd8K/W5lmyeuRT/J7Vflo71x0Cp15RfTHZSdiPZSeaFvPmy15nAgjR1dFaJfeb9NdnvaSvwFeDmbtq9DfgS2Sh6K/BhsstxAf4DeAzYKOnZMva9key5/D1wK3BhekcI2Qn0P5El9wWpPm8m3Rx3RPyJLMmfRvZO69vA5FzfVgfKpmnNSiPpp2QnGG+sdyz1JOkfgYMjosurbMz6K4/0zUqQpsnenaaUjieb5rir3nGZlcufyDUrzT5kUzqHkk11XA3cXdeIzCrg6R0zswLx9I6ZWYH0++md4cOHR1NTU73DMDMbMJYvX/5sRDR0Vdfvk35TUxMtLS31DsPMbMCQ1OUnusHTO2ZmheKkb2ZWIE76ZmYF0u/n9M3MetPrr79Oa2srr776ar1DKdvQoUNpbGxkyJAhJW/jpG9mhdba2so+++xDU1MTf76HT/8XEWzZsoXW1lZGjx5d8nae3jGzQnv11Vc58MADB1TCB5DEgQceWPY7FCd9Myu8gZbwO1QSt5O+mVmBeE7fzCyn6bJ7atrfutkfq2l/1XLSt5JU+o/Q3/7gzYrO0ztmZnV2yy23cPzxxzN+/HguuOACtm/fzt57782Xv/xljjnmGCZMmMCmTZtqsq8eR/rpnqo3AyPIbrA8NyKuk3QA2a3ymoB1wFkR8Vy6efZ1wOlk98ecGhGPpL6mAFekrq+KiAU1OQrrddMHL6pwS4/0zXZm9erV3H777fz85z9nyJAhfP7zn+fWW2/lpZdeYsKECcyaNYtLL72UG264gSuuuKLnDntQyvTONuCSiHgk3Xx6uaQlZPclXRoRsyVdBlxGdg/S08jupTqG7J6fc4D3pReJrwDNZC8eyyUtjojnqj4KM7MBaunSpSxfvpz3vve9ALzyyiscdNBB7LbbbkycOBGA4447jiVLltRkfz1O70REW8dIPSL+AKwGRgKTyG6UTPp5RlqeBNwcmWXA/pIOAT4KLImIrSnRLwFOrclRmJkNUBHBlClTWLFiBStWrGDNmjXMnDmTIUOGvHFJ5qBBg9i2bVtN9lfWnL6kJuBY4CFgRES0paqNZNM/kL0grM9t1prKuivvaj/TJLVIamlvby8nRDOzAeXkk09m0aJFbN68GYCtW7fyzDPdfjNy1Uq+ekfS3sCdwPSIeDH/oYCICEk1u+9iRMwF5gI0Nzf7fo5m1mf6+oqzo446iquuuopTTjmFHTt2MGTIEK6//vpe219JSV/SELKEf2tEfD8Vb5J0SES0pembzal8AzAqt3ljKtsAnNip/KeVh25mtms4++yzOfvss99U9sc//vGN5TPPPJMzzzyzJvvqcXonXY1zE7A6Iv45V7UYmJKWpwB358onKzMBeCFNA90HnCJpmKRhwCmpzMzM+kgpI/0PAJ8GHpW0IpVdDswG7pB0PvAMcFaqu5fscs21ZJdsngcQEVslXQk8nNp9NSK21uQozMysJD0m/Yh4EOjuW31O7qJ9ABd109c8YF45AZqZWe34E7lmZgXipG9mViBO+mZmBeJv2TQzy/vJ12rb30kzattflTzSNzPrRyKCHTt29Fr/TvpmZnW2bt06xo4dy+TJkxk3bhyDBg16o27RokVMnTq1Zvvy9E7BXHvFefUOwcy68MQTT7BgwQImTJjA3nvv3Wv78UjfzKwfOPzww5kwYUKv78dJ38ysH9hrr73eWM5/oeWrr75a0/046ZuZ9TMjRoxg9erV7Nixg7vuuqumfXtO38wsrx9cYjl79mwmTpxIQ0MDzc3Nb/rGzWo56ZuZ1VlTUxOrVq16Y72WX6Xcmad3zMwKxEnfzKxAnPTNrPCyb4QfeCqJ20nfzApt6NChbNmyZcAl/ohgy5YtDB06tKztejyRK2keMBHYHBHjUtntwNjUZH/g+YgYL6kJWA2sSXXLIuLCtM1xwHxgD7K7a/1tDLRn2cx2OY2NjbS2ttLe3l7vUMo2dOhQGhsby9qmlKt35gPfAm7uKIiIN+7gK+lq4IVc+ycjYnwX/cwBPgc8RJb0TwV+WFa0ZmY1NmTIEEaPHl3vMPpMj9M7EfEA0OW9bNNN088CFu6sD0mHAPtGxLI0ur8ZOKP8cM3MrBrVzumfAGyKiCdyZaMl/UrSzySdkMpGAq25Nq2prEuSpklqkdQyEN9ymZn1V9Um/XN58yi/DTgsIo4FvgR8T9K+5XYaEXMjojkimhsaGqoM0czMOlT8iVxJg4FPAsd1lEXEa8BraXm5pCeBI4ENQP5sQ2MqMzOzPlTNSP8vgccj4o1pG0kNkgal5bcDY4CnIqINeFHShHQeYDJwdxX7NjOzCvSY9CUtBH4BjJXUKun8VHUObz2B+yFgpaQVwCLgwojoOAn8eeBGYC3wJL5yx8ysz/U4vRMR53ZTPrWLsjuBO7tp3wKMKzM+MzOrIX8i18ysQJz0zcwKxEnfzKxAnPTNzArESd/MrECc9M3MCsRJ38ysQJz0zcwKxEnfzKxAnPTNzArESd/MrECc9M3MCsRJ38ysQJz0zcwKxEnfzKxAnPTNzAqklDtnzZO0WdKqXNlMSRskrUiP03N1MyStlbRG0kdz5aemsrWSLqv9oZiZWU9KGenPB07tovyaiBifHvcCSDqK7DaKR6dtvi1pULpv7vXAacBRwLmprZmZ9aFSbpf4gKSmEvubBNwWEa8BT0taCxyf6tZGxFMAkm5LbX9TdsRmZlaxaub0L5a0Mk3/DEtlI4H1uTatqay78i5JmiapRVJLe3t7FSGamVlepUl/DnAEMB5oA66uWURARMyNiOaIaG5oaKhl12Zmhdbj9E5XImJTx7KkG4AfpNUNwKhc08ZUxk7Kzcysj1Q00pd0SG71E0DHlT2LgXMk7S5pNDAG+CXwMDBG0mhJu5Gd7F1cedhmZlaJHkf6khYCJwLDJbUCXwFOlDQeCGAdcAFARDwm6Q6yE7TbgIsiYnvq52LgPmAQMC8iHqv50ZiZ2U6VcvXOuV0U37ST9rOAWV2U3wvcW1Z0ZmZWUxXN6Zv1pmuvOK+i7aZf9d0aR2K26/HXMJiZFYiTvplZgTjpm5kViJO+mVmBOOmbmRWIk76ZWYE46ZuZFYiTvplZgTjpm5kViJO+mVmBOOmbmRWIk76ZWYE46ZuZFYiTvplZgTjpm5kVSI9JX9I8SZslrcqV/ZOkxyWtlHSXpP1TeZOkVyStSI/v5LY5TtKjktZK+qYk9c4hmZlZd0oZ6c8HTu1UtgQYFxHvBn4LzMjVPRkR49Pjwlz5HOBzZPfNHdNFn2Zm1st6TPoR8QCwtVPZ/RGxLa0uAxp31ke6kfq+EbEsIgK4GTijspDNzKxStZjT/wzww9z6aEm/kvQzSSekspFAa65NayrrkqRpkloktbS3t9cgRDMzgyqTvqQvA9uAW1NRG3BYRBwLfAn4nqR9y+03IuZGRHNENDc0NFQTopmZ5VR8Y3RJU4GJwMlpyoaIeA14LS0vl/QkcCSwgTdPATWmMjMz60MVjfQlnQpcCnw8Il7OlTdIGpSW3052wvapiGgDXpQ0IV21Mxm4u+rozcysLD2O9CUtBE4EhktqBb5CdrXO7sCSdOXlsnSlzoeAr0p6HdgBXBgRHSeBP092JdAeZOcA8ucBzMysD/SY9CPi3C6Kb+qm7Z3And3UtQDjyorOzMxqyp/INTMrECd9M7MCcdI3MysQJ30zswJx0jczKxAnfTOzAnHSNzMrECd9M7MCcdI3MysQJ30zswJx0jczKxAnfTOzAnHSNzMrECd9M7MCcdI3MysQJ30zswJx0jczK5CSkr6keZI2S1qVKztA0hJJT6Sfw1K5JH1T0lpJKyW9J7fNlNT+CUlTan84Zma2M6WO9OcDp3YquwxYGhFjgKVpHeA0shuijwGmAXMge5Egu7/u+4Djga90vFCYmVnfKCnpR8QDwNZOxZOABWl5AXBGrvzmyCwD9pd0CPBRYElEbI2I54AlvPWFxMzMelE1c/ojIqItLW8ERqTlkcD6XLvWVNZd+VtImiapRVJLe3t7FSGamVleTU7kRkQAUYu+Un9zI6I5IpobGhpq1a2ZWeENrmLbTZIOiYi2NH2zOZVvAEbl2jWmsg3AiZ3Kf1rF/ncNP/laZdudNKO2cZhZIVQz0l8MdFyBMwW4O1c+OV3FMwF4IU0D3QecImlYOoF7SiozM7M+UtJIX9JCslH6cEmtZFfhzAbukHQ+8AxwVmp+L3A6sBZ4GTgPICK2SroSeDi1+2pEdD45bGZmvaikpB8R53ZTdXIXbQO4qJt+5gHzSo7OzMxqyp/INTMrkGpO5FoNXLv0txVtN/2kGgdiZoXgkb6ZWYE46ZuZFYiTvplZgTjpm5kViJO+mVmBOOmbmRWIk76ZWYH4On0rvKbL7qlou3WzP1bjSMx6n0f6ZmYF4pG+Fd70wYsq3NIjfRt4PNI3MysQJ30zswJx0jczKxAnfTOzAqk46UsaK2lF7vGipOmSZkrakCs/PbfNDElrJa2R9NHaHIKZmZWq4qt3ImINMB5A0iCyG5/fRXZ7xGsi4hv59pKOAs4BjgYOBX4s6ciI2F5pDGZmVp5aTe+cDDwZEc/spM0k4LaIeC0inia7h+7xNdq/mZmVoFZJ/xxgYW79YkkrJc2TNCyVjQTW59q0prK3kDRNUouklvb29hqFaGZmVSd9SbsBHwf+NRXNAY4gm/ppA64ut8+ImBsRzRHR3NDQUG2IZmaW1GKkfxrwSERsAoiITRGxPSJ2ADfw5ymcDcCo3HaNqczMzPpILZL+ueSmdiQdkqv7BLAqLS8GzpG0u6TRwBjglzXYv5mZlaiq796RtBfwEeCCXPHXJY0HAljXURcRj0m6A/gNsA24yFfumJn1raqSfkS8BBzYqezTO2k/C5hVzT7NzKxy/kSumVmBOOmbmRWIk76ZWYE46ZuZFYiTvplZgTjpm5kViJO+mVmBOOmbmRWIk76ZWYE46ZuZFYiTvplZgTjpm5kViJO+mVmBOOmbmRWIk76ZWYE46ZuZFUgtboy+TtKjklZIakllB0haIumJ9HNYKpekb0paK2mlpPdUu38zMytdrUb6J0XE+IhoTuuXAUsjYgywNK1DdhP1MekxDZhTo/2bmVkJemt6ZxKwIC0vAM7Ild8cmWXA/p1upG5mZr2oFkk/gPslLZc0LZWNiIi2tLwRGJGWRwLrc9u2prI3kTRNUouklvb29hqEaGZmUOWN0ZMPRsQGSQcBSyQ9nq+MiJAU5XQYEXOBuQDNzc1lbWtmZt2reqQfERvSz83AXcDxwKaOaZv0c3NqvgEYldu8MZWZmVkfqCrpS9pL0j4dy8ApwCpgMTAlNZsC3J2WFwOT01U8E4AXctNAZmbWy6qd3hkB3CWpo6/vRcSPJD0M3CHpfOAZ4KzU/l7gdGAt8DJwXpX7NzOzMlSV9CPiKeCYLsq3ACd3UR7ARdXs08zMKudP5JqZFYiTvplZgTjpm5kViJO+mVmBOOmbmRWIk76ZWYE46ZuZFYiTvplZgTjpm5kVSC2+ZdOskJouu6ei7dbN/liNIzErnUf6ZmYF4pG+WYWmD15U4ZYe6Vv9eKRvZlYgTvpmZgXipG9mViBO+mZmBVLxiVxJo4Cbye6eFcDciLhO0kzgc0B7anp5RNybtpkBnA9sB74YEfdVEXvv+cnXyt/mpBm1j8PMrMaquXpnG3BJRDyS7pO7XNKSVHdNRHwj31jSUcA5wNHAocCPJR0ZEduriKFXXLv0t2VvM/2kXgjEzKzGKp7eiYi2iHgkLf8BWA2M3Mkmk4DbIuK1iHia7D65x1e6fzMzK19N5vQlNQHHAg+looslrZQ0T9KwVDYSWJ/brJWdv0iYmVmNVZ30Je0N3AlMj4gXgTnAEcB4oA24uoI+p0lqkdTS3t7e8wZmZlaSqpK+pCFkCf/WiPg+QERsiojtEbEDuIE/T+FsAEblNm9MZW8REXMjojkimhsaGqoJ0czMcipO+pIE3ASsjoh/zpUfkmv2CWBVWl4MnCNpd0mjgTHALyvdv5mZla+aq3c+AHwaeFTSilR2OXCupPFkl3GuAy4AiIjHJN0B/Ibsyp+L+uOVO2Zmu7KKk35EPAioi6p7d7LNLGBWpfs0M7Pq+BO5ZmYF4qRvZlYgTvpmZgXipG9mViBO+mZmBeKkb2ZWIE76ZmYF4qRvZlYg1Xwi18z6UNNl95S9zbrZH+uFSGwgc9I3GyCmD15UwVZO+vZmnt4xMysQJ30zswJx0jczKxAnfTOzAnHSNzMrECd9M7MCcdI3MyuQPr9OX9KpwHXAIODGiJjd1zGY2c5de8V5FW03/arv1jgSq7U+TfqSBgHXAx8BWoGHJS2OiN/0xv4q/cM1M9tV9fVI/3hgbUQ8BSDpNmAS2c3SzWyA6+t3CH5HUj5FRN/tTDoTODUiPpvWPw28LyIu7tRuGjAtrY4F1vRZkKUbDjxb7yAq5Njrw7H3vYEaN1QX++ER0dBVRb/87p2ImAvMrXccOyOpJSKa6x1HJRx7fTj2vjdQ44bei72vr97ZAIzKrTemMjMz6wN9nfQfBsZIGi1pN+AcYHEfx2BmVlh9Or0TEdskXQzcR3bJ5ryIeKwvY6ihfj391APHXh+Ove8N1Lihl2Lv0xO5ZmZWX/5ErplZgTjpm5kViJN+hSQNkvQrST+odyzlkLS/pEWSHpe0WtL76x1TKST9L0mPSVolaaGkofWOqTuS5knaLGlVruwASUskPZF+DqtnjN3pJvZ/Sn8vKyXdJWn/esbYna5iz9VdIikkDa9HbD3pLnZJX0jP/WOSvl6LfTnpV+5vgdX1DqIC1wE/ioh3AscwAI5B0kjgi0BzRIwjuwjgnPpGtVPzgVM7lV0GLI2IMcDStN4fzeetsS8BxkXEu4HfAjP6OqgSzeetsSNpFHAK8Lu+DqgM8+kUu6STyL6x4JiIOBr4Ri125KRfAUmNZHecvrHesZRD0n7Ah4CbACLiTxHxfH2jKtlgYA9Jg4E9gd/XOZ5uRcQDwNZOxZOABWl5AXBGnwZVoq5ij4j7I2JbWl1G9vmafqeb5x3gGuBSoN9etdJN7H8DzI6I11KbzbXYl5N+Za4l+yPaUe9AyjQaaAe+m6ambpS0V72D6klEbCAb5fwOaANeiIj76xtV2UZERFta3giMqGcwVfgM8MN6B1EqSZOADRHx63rHUoEjgRMkPSTpZ5LeW4tOnfTLJGkisDkiltc7lgoMBt4DzImIY4GX6L/TDG9I89+TyF60DgX2kvQ/6xtV5SK7Trrfjjq7I+nLwDbg1nrHUgpJewKXA39f71gqNBg4AJgA/G/gDkmqtlMn/fJ9APi4pHXAbcB/k3RLfUMqWSvQGhEPpfVFZC8C/d1fAk9HRHtEvA58H/iLOsdUrk2SDgFIP2vyVr2vSJoKTAQ+FQPnwz1HkA0Ufp3+XxuBRyQdXNeoStcKfD8yvySbWaj6RLSTfpkiYkZENEZEE9nJxP+IiAEx6oyIjcB6SWNT0ckMjK+1/h0wQdKeaaRzMgPgBHQni4EpaXkKcHcdYylLuvHRpcDHI+LlesdTqoh4NCIOioim9P/aCrwn/R8MBP8GnAQg6UhgN2rwjaFO+sXzBeBWSSuB8cD/rXM8PUrvTBYBjwCPkv3d9tuP10taCPwCGCupVdL5wGzgI5KeIHvn0i/vGNdN7N8C9gGWSFoh6Tt1DbIb3cQ+IHQT+zzg7ekyztuAKbV4l+WvYTAzKxCP9M3MCsRJ38ysQJz0zcwKxEnfzKxAnPTNzArESd/MrECc9M3MCuT/AyL9Cf/Kr85xAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YBebFSFfio15",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 29
        },
        "outputId": "b7404ae2-a0cd-44c9-bfb8-c54b37bfd776"
      },
      "source": [
        "all_tokens_set_en = set()\n",
        "for name in names:\n",
        "    all_tokens_set_en.update(set(name))\n",
        "\n",
        "\n",
        "tokens_en = list(all_tokens_set_en)# <list of all unique characters in the dataset>\n",
        "\n",
        "num_tokens_en = len(tokens_en)\n",
        "print ('num_tokens = ', num_tokens_en)\n",
        "\n",
        "assert 50 < num_tokens_en < 60, \"Names should contain within 50 and 60 unique tokens depending on encoding\""
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "num_tokens =  55\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u6_IkBBsi0eE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 29
        },
        "outputId": "aba262ca-cfc0-4faf-f224-4481a48416d3"
      },
      "source": [
        "all_tokens_set_ru = set()\n",
        "for name in names_ru:\n",
        "    all_tokens_set_ru.update(set(name))\n",
        "\n",
        "\n",
        "tokens_ru = list(all_tokens_set_ru)# <list of all unique characters in the dataset>\n",
        "\n",
        "num_tokens_ru = len(tokens_ru)\n",
        "print ('num_tokens = ', num_tokens_ru)\n",
        "\n",
        "assert 50 < num_tokens_ru < 60, \"Names should contain within 50 and 60 unique tokens depending on encoding\""
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "num_tokens =  54\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Prwc6k2ji3me",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "token_to_id_en = {\n",
        "    token: idx for idx, token in enumerate(tokens_en)\n",
        "}\n",
        "\n",
        "token_to_id_ru = {\n",
        "    token: idx for idx, token in enumerate(tokens_ru)\n",
        "}"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AxOZM_y_i6pU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 29
        },
        "outputId": "d8efc911-6bda-4a9e-fc2e-3cb6559bafe6"
      },
      "source": [
        "assert len(tokens_ru) == len(token_to_id_ru), \"dictionaries must have same size\"\n",
        "\n",
        "for i in range(num_tokens_ru):\n",
        "    assert token_to_id_ru[tokens_ru[i]] == i, \"token identifier must be it's position in tokens list\"\n",
        "\n",
        "for i in range(num_tokens_en):\n",
        "    assert token_to_id_en[tokens_en[i]] == i, \"token identifier must be it's position in tokens list\"\n",
        "\n",
        "print(\"Seems alright!\")"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Seems alright!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "djDcoVhfi-z4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def to_matrix(names, token_to_id, max_len=None, pad=None, dtype='int32', batch_first=False):\n",
        "    \"\"\"Casts a list of names into rnn-digestable matrix\"\"\"\n",
        "    pad = token_to_id[' ']\n",
        "    max_len = max_len or max(map(len, names))\n",
        "    names_ix = np.zeros([len(names), max_len], dtype) + pad\n",
        "\n",
        "    for i in range(len(names)):\n",
        "        line_ix = [token_to_id[c] for c in names[i]]\n",
        "        names_ix[i, :len(line_ix)] = line_ix\n",
        "        \n",
        "    if not batch_first: # convert [batch, time] into [time, batch]\n",
        "        names_ix = np.transpose(names_ix)\n",
        "\n",
        "    return names_ix"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iu0YDkBhjDOZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch, torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NNY9-9k_jGa0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyModel(nn.Module):\n",
        "    def __init__(self, num_tokens=num_tokens_en, emb_size=16, rnn_num_units=64):\n",
        "        super(self.__class__, self).__init__()\n",
        "        self.emb = nn.Embedding(num_tokens, emb_size)\n",
        "        self.rnn = nn.RNN(emb_size, rnn_num_units)\n",
        "        self.hid_to_logits = nn.Linear(rnn_num_units, num_tokens)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        assert isinstance(x.data, torch.LongTensor)\n",
        "        h_seq, _ = self.rnn(self.emb(x))\n",
        "        next_logits = self.hid_to_logits(h_seq)\n",
        "        next_logp = F.log_softmax(next_logits, dim=-1)\n",
        "        return next_logp"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avXqd4qsjKVZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = MyModel()\n",
        "opt = torch.optim.Adam(model.parameters())\n",
        "criterion = nn.NLLLoss()\n",
        "history = []\n",
        "\n",
        "# the model applies over the whole sequence\n",
        "batch_ix = to_matrix(sample(names, 32), token_to_id_en, max_len=MAX_LENGTH)\n",
        "batch_ix = torch.LongTensor(batch_ix)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M7cYL66mjNkW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logp_seq = model(batch_ix)\n",
        "\n",
        "loss = criterion(logp_seq[:, :-1].contiguous().view(-1, num_tokens_en),\n",
        "                 batch_ix[:, 1:].contiguous().view(-1))\n",
        "\n",
        "loss.backward()"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wGZuDE3SjRRv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.tensorboard import SummaryWriter\n",
        "\n",
        "# default `log_dir` is \"runs\" - we'll be more specific here\n",
        "writer = SummaryWriter()"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8J6-7jyCjTxG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "writer.add_graph(model, batch_ix)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "73u0dU0XjWAb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LENGTH = 16\n",
        "\n",
        "for i in range(1000):\n",
        "    batch_ix = to_matrix(sample(names, 32), token_to_id_en, max_len=MAX_LENGTH)\n",
        "    batch_ix = torch.tensor(batch_ix, dtype=torch.int64)\n",
        "    \n",
        "    logp_seq = model(batch_ix)\n",
        "    \n",
        "    loss = criterion(logp_seq[:, :-1].contiguous().view(-1, num_tokens_en),\n",
        "                 batch_ix[:, 1:].contiguous().view(-1))\n",
        "\n",
        "    loss.backward()\n",
        "    \n",
        "    opt.step()\n",
        "    opt.zero_grad()\n",
        "    \n",
        "    history.append(loss.data.numpy())\n",
        "    if (i+1)%100==0:\n",
        "        writer.add_scalar('train loss', history[-1], i)\n",
        "\n",
        "assert np.mean(history[:10]) > np.mean(history[-10:]), \"RNN didn't converge.\""
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b8p3_qjdjzEc",
        "colab_type": "text"
      },
      "source": [
        "# Char-Level Machine Translation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "05hC7XPXuN7h",
        "colab_type": "text"
      },
      "source": [
        "    Task: Translate from English to Russian"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rz_TNkeTjbmW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, \n",
        "                 num_tokens=num_tokens_en, \n",
        "                 emb_size=16, \n",
        "                 rnn_num_units=64):\n",
        "        super(self.__class__, self).__init__()\n",
        "        self.emb = nn.Embedding(num_tokens, emb_size)\n",
        "        self.rnn = nn.RNN(emb_size, rnn_num_units)\n",
        "        self.hid_to_logits = nn.Linear(rnn_num_units, num_tokens)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        assert isinstance(x.data, torch.LongTensor)\n",
        "        h_seq, h_last = self.rnn(self.emb(x))\n",
        "        next_logits = self.hid_to_logits(h_seq)\n",
        "        next_logp = F.log_softmax(next_logits, dim=-1)\n",
        "        return next_logp, h_last"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bisbjuSGudRZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, \n",
        "                 num_tokens=num_tokens_ru, \n",
        "                 emb_size=16, \n",
        "                 rnn_num_units=64):\n",
        "        super(self.__class__, self).__init__()\n",
        "        self.emb = nn.Embedding(num_tokens, emb_size)\n",
        "        self.rnn = nn.RNN(emb_size, rnn_num_units)\n",
        "        self.hid_to_logits = nn.Linear(rnn_num_units, num_tokens)\n",
        "        \n",
        "    def forward(self, x, enc_last_state):\n",
        "        assert isinstance(x.data, torch.LongTensor)\n",
        "        h_seq, h_last = self.rnn(self.emb(x), enc_last_state)\n",
        "        next_logits = self.hid_to_logits(h_seq)\n",
        "        next_logp = F.log_softmax(next_logits, dim=-1)\n",
        "        return next_logp, h_last"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ArBfqFTMulE-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self,\n",
        "                 num_tokens_en = num_tokens_en,\n",
        "                 emb_size = 16,\n",
        "                 rnn_num_units = 64):\n",
        "        super(self.__class__, self).__init__()\n",
        "        self.emb = nn.Embedding(num_tokens_en, emb_size)\n",
        "        self.rnn = nn.RNN(emb_size, rnn_num_units)\n",
        "        self.hid_to_logits = nn.Linear(rnn_num_units, num_tokens_en)\n",
        "    \n",
        "    def forward(self, src):\n",
        "        h_seq, _ = self.rnn(self.emb(src))\n",
        "        next_logit = self.hid_to_logits(h_seq)\n",
        "        logp_seq = F.log_softmax(next_logit, dim = -1) \n",
        "        \n",
        "        return logp_seq"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wsNSsLCJwA8l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Seq2Seq()\n",
        "opt = torch.optim.SGD(model.parameters(), lr = 0.002)\n",
        "loss_function = nn.NLLLoss()"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pvG_ijg3wLsD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "indices = np.random.choice(np.arange(len(names)), size=32)\n",
        "batch_en = to_matrix(np.array(names)[indices], token_to_id=token_to_id_en, max_len=MAX_LENGTH)\n",
        "input_tensor = torch.from_numpy(batch_en).type(torch.int64)\n",
        "\n",
        "batch_ru = to_matrix(np.array(names_ru)[indices], token_to_id=token_to_id_ru, max_len=MAX_LENGTH)\n",
        "target_tensor = torch.from_numpy(batch_ru).type(torch.int64)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pTuong6BwXcd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "out = model(input_tensor)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B41RY1Pqwaul",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "idx_to_token_en = {idx: token for token, idx in token_to_id_en.items()}\n",
        "idx_to_token_ru = {idx: token for token, idx in token_to_id_ru.items()}"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "huWeQPYMwfRC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = out.argmax(dim=-1)"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vOwSr159whHn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ------------- Train model\n",
        "MAX_LENGTH = 16\n",
        "\n",
        "for i in range(1000):\n",
        "    batch_ix = to_matrix(sample(names, 32), token_to_id_en, max_len=MAX_LENGTH)\n",
        "    batch_ix = torch.tensor(batch_ix, dtype=torch.int64)\n",
        "    \n",
        "    logp_seq = model(batch_ix)\n",
        "    \n",
        "    loss = criterion(logp_seq[:, :-1].contiguous().view(-1, num_tokens_en),\n",
        "                 batch_ix[:, 1:].contiguous().view(-1))\n",
        "\n",
        "    loss.backward()\n",
        "    \n",
        "    opt.step()\n",
        "    opt.zero_grad()\n",
        "    \n",
        "    history.append(loss.data.numpy())\n",
        "    if (i+1)%100==0:\n",
        "        writer.add_scalar('train loss', history[-1], i)"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4HlWInA0wn-0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_example(idx):\n",
        "    translated = ''.join([idx_to_token_ru[x] for x in a[:, idx].detach().numpy()])\n",
        "    original = ''.join([idx_to_token_en[x] for x in input_tensor[:, idx].detach().numpy()])\n",
        "    print(original, translated)"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qF3LRkpEw5yr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 29
        },
        "outputId": "6b9de20c-fa77-4d7c-b937-cbe7bf38fb03"
      },
      "source": [
        "get_example(12)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Darin            НфбНЗ          \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HyznivF7w9Qe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}