nlp_datasets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP datasets.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNeieoZrIHg42+GOPAc0O7m",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/howard-haowen/NLP-demos/blob/main/nlp_datasets.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Scikit-learn"
      ],
      "metadata": {
        "id": "s0TTjg1ctVtR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 20 newsgroups"
      ],
      "metadata": {
        "id": "gzc_J4WTtj6y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.datasets import fetch_20newsgroups"
      ],
      "metadata": {
        "id": "ObiMdb0ftwWX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "- `subset`: {`train`, `test`, `all`}\n",
        "- `remove`: any subset of (`headers`, `footers`, `quotes`) to remove metadata\n",
        "- `categories`: a list of category names"
      ],
      "metadata": {
        "id": "QD1rjoR5umCD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "newsgroups = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))"
      ],
      "metadata": {
        "id": "5Y3DHojRuhKe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cat_names = newsgroups.target_names\n",
        "cat_names"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QAOBA5nbvUN3",
        "outputId": "04aa1ece-97ec-4902-de6c-db9e4d7e3d5e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['alt.atheism',\n",
              " 'comp.graphics',\n",
              " 'comp.os.ms-windows.misc',\n",
              " 'comp.sys.ibm.pc.hardware',\n",
              " 'comp.sys.mac.hardware',\n",
              " 'comp.windows.x',\n",
              " 'misc.forsale',\n",
              " 'rec.autos',\n",
              " 'rec.motorcycles',\n",
              " 'rec.sport.baseball',\n",
              " 'rec.sport.hockey',\n",
              " 'sci.crypt',\n",
              " 'sci.electronics',\n",
              " 'sci.med',\n",
              " 'sci.space',\n",
              " 'soc.religion.christian',\n",
              " 'talk.politics.guns',\n",
              " 'talk.politics.mideast',\n",
              " 'talk.politics.misc',\n",
              " 'talk.religion.misc']"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cat_idxes = newsgroups.target\n",
        "len(cat_idxes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3GnVgj5Svhap",
        "outputId": "3a5b45c9-7be0-4e85-a26a-e781502c61e4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "18846"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "news = newsgroups.data\n",
        "len(news)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NpE86zawvprX",
        "outputId": "1bb91b65-a6d3-4339-878a-e9db73b51854"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "18846"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "news[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        },
        "id": "gew61QyjxD--",
        "outputId": "f1fd941c-2b99-49a4-f2b7-9452e1b20b82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"\\n\\nI am sure some bashers of Pens fans are pretty confused about the lack\\nof any kind of posts about the recent Pens massacre of the Devils. Actually,\\nI am  bit puzzled too and a bit relieved. However, I am going to put an end\\nto non-PIttsburghers' relief with a bit of praise for the Pens. Man, they\\nare killing those Devils worse than I thought. Jagr just showed you why\\nhe is much better than his regular season stats. He is also a lot\\nfo fun to watch in the playoffs. Bowman should let JAgr have a lot of\\nfun in the next couple of games since the Pens are going to beat the pulp out of Jersey anyway. I was very disappointed not to see the Islanders lose the final\\nregular season game.          PENS RULE!!!\\n\\n\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "def newsgroups2df(subset='all',\n",
        "                  remove=('headers', 'footers', 'quotes')):\n",
        "    newsgroups = fetch_20newsgroups(subset=subset, remove=remove)\n",
        "    cat_names = newsgroups.target_names\n",
        "    cat_idxes = newsgroups.target\n",
        "    news = newsgroups.data\n",
        "    df = pd.DataFrame(\n",
        "        {\n",
        "            \"news\": news,\n",
        "            \"cat\": [cat_names[c] for c in cat_idxes],\n",
        "        }\n",
        "    )\n",
        "    return df"
      ],
      "metadata": {
        "id": "rQOfZzN1vzk1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_train = newsgroups2df(subset='train')\n",
        "df_train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 467
        },
        "id": "re7CFngBvt7f",
        "outputId": "7a3439a5-8545-4580-d98b-14a3fa256853"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    news  \\\n",
              "0      I was wondering if anyone out there could enli...   \n",
              "1      A fair number of brave souls who upgraded thei...   \n",
              "2      well folks, my mac plus finally gave up the gh...   \n",
              "3      \\nDo you have Weitek's address/phone number?  ...   \n",
              "4      From article <C5owCB.n3p@world.std.com>, by to...   \n",
              "...                                                  ...   \n",
              "11309  DN> From: nyeda@cnsvax.uwec.edu (David Nye)\\nD...   \n",
              "11310  I have a (very old) Mac 512k and a Mac Plus, b...   \n",
              "11311  I just installed a DX2-66 CPU in a clone mothe...   \n",
              "11312  \\nWouldn't this require a hyper-sphere.  In 3-...   \n",
              "11313  Stolen from Pasadena between 4:30 and 6:30 pm ...   \n",
              "\n",
              "                            cat  \n",
              "0                     rec.autos  \n",
              "1         comp.sys.mac.hardware  \n",
              "2         comp.sys.mac.hardware  \n",
              "3                 comp.graphics  \n",
              "4                     sci.space  \n",
              "...                         ...  \n",
              "11309                   sci.med  \n",
              "11310     comp.sys.mac.hardware  \n",
              "11311  comp.sys.ibm.pc.hardware  \n",
              "11312             comp.graphics  \n",
              "11313           rec.motorcycles  \n",
              "\n",
              "[11314 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-66ab9b58-6a05-4a94-975f-e8e52d88d550\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>news</th>\n",
              "      <th>cat</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I was wondering if anyone out there could enli...</td>\n",
              "      <td>rec.autos</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A fair number of brave souls who upgraded thei...</td>\n",
              "      <td>comp.sys.mac.hardware</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>well folks, my mac plus finally gave up the gh...</td>\n",
              "      <td>comp.sys.mac.hardware</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\\nDo you have Weitek's address/phone number?  ...</td>\n",
              "      <td>comp.graphics</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>From article &lt;C5owCB.n3p@world.std.com&gt;, by to...</td>\n",
              "      <td>sci.space</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11309</th>\n",
              "      <td>DN&gt; From: nyeda@cnsvax.uwec.edu (David Nye)\\nD...</td>\n",
              "      <td>sci.med</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11310</th>\n",
              "      <td>I have a (very old) Mac 512k and a Mac Plus, b...</td>\n",
              "      <td>comp.sys.mac.hardware</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11311</th>\n",
              "      <td>I just installed a DX2-66 CPU in a clone mothe...</td>\n",
              "      <td>comp.sys.ibm.pc.hardware</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11312</th>\n",
              "      <td>\\nWouldn't this require a hyper-sphere.  In 3-...</td>\n",
              "      <td>comp.graphics</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11313</th>\n",
              "      <td>Stolen from Pasadena between 4:30 and 6:30 pm ...</td>\n",
              "      <td>rec.motorcycles</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11314 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-66ab9b58-6a05-4a94-975f-e8e52d88d550')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-66ab9b58-6a05-4a94-975f-e8e52d88d550 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-66ab9b58-6a05-4a94-975f-e8e52d88d550');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_test = newsgroups2df(subset='test')\n",
        "df_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "0svj72gPxoHn",
        "outputId": "55817d07-c86a-46ba-92f4-96bd842e2baf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                   news  \\\n",
              "0     I am a little confused on all of the models of...   \n",
              "1     I'm not familiar at all with the format of the...   \n",
              "2                                   \\nIn a word, yes.\\n   \n",
              "3     \\nThey were attacking the Iraqis to drive them...   \n",
              "4     \\nI've just spent two solid months arguing tha...   \n",
              "...                                                 ...   \n",
              "7527  \\n   Henry, if I read you correctly, you may b...   \n",
              "7528  about\\nthem on\\n\\nActually, I thought Macs wer...   \n",
              "7529  I sent a version of this post out a while ago,...   \n",
              "7530  I have this kit which includes the following :...   \n",
              "7531  \\nFine, but one of the points of this entire d...   \n",
              "\n",
              "                         cat  \n",
              "0                  rec.autos  \n",
              "1             comp.windows.x  \n",
              "2                alt.atheism  \n",
              "3      talk.politics.mideast  \n",
              "4         talk.religion.misc  \n",
              "...                      ...  \n",
              "7527               sci.space  \n",
              "7528   comp.sys.mac.hardware  \n",
              "7529      rec.sport.baseball  \n",
              "7530            misc.forsale  \n",
              "7531  soc.religion.christian  \n",
              "\n",
              "[7532 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-264fdef4-a4b3-4781-86b3-6f0745b01a55\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>news</th>\n",
              "      <th>cat</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I am a little confused on all of the models of...</td>\n",
              "      <td>rec.autos</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I'm not familiar at all with the format of the...</td>\n",
              "      <td>comp.windows.x</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\\nIn a word, yes.\\n</td>\n",
              "      <td>alt.atheism</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\\nThey were attacking the Iraqis to drive them...</td>\n",
              "      <td>talk.politics.mideast</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\\nI've just spent two solid months arguing tha...</td>\n",
              "      <td>talk.religion.misc</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7527</th>\n",
              "      <td>\\n   Henry, if I read you correctly, you may b...</td>\n",
              "      <td>sci.space</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7528</th>\n",
              "      <td>about\\nthem on\\n\\nActually, I thought Macs wer...</td>\n",
              "      <td>comp.sys.mac.hardware</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7529</th>\n",
              "      <td>I sent a version of this post out a while ago,...</td>\n",
              "      <td>rec.sport.baseball</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7530</th>\n",
              "      <td>I have this kit which includes the following :...</td>\n",
              "      <td>misc.forsale</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7531</th>\n",
              "      <td>\\nFine, but one of the points of this entire d...</td>\n",
              "      <td>soc.religion.christian</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7532 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-264fdef4-a4b3-4781-86b3-6f0745b01a55')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-264fdef4-a4b3-4781-86b3-6f0745b01a55 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-264fdef4-a4b3-4781-86b3-6f0745b01a55');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Gensim"
      ],
      "metadata": {
        "id": "_0zR-UrKmz2i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import gensim.downloader as api"
      ],
      "metadata": {
        "id": "rKbBHVV6m1f_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "See [here](https://github.com/RaRe-Technologies/gensim-data) for more details."
      ],
      "metadata": {
        "id": "H7vMApfUoRGb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "corpora = list(api.info()['corpora'].keys())\n",
        "corpora"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "py-shPOvnMVu",
        "outputId": "d7a73f0a-47a6-42e3-9a11-b883f7522ff7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['semeval-2016-2017-task3-subtaskBC',\n",
              " 'semeval-2016-2017-task3-subtaskA-unannotated',\n",
              " 'patent-2017',\n",
              " 'quora-duplicate-questions',\n",
              " 'wiki-english-20171001',\n",
              " 'text8',\n",
              " 'fake-news',\n",
              " '20-newsgroups',\n",
              " '__testing_matrix-synopsis',\n",
              " '__testing_multipart-matrix-synopsis']"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "models = list(api.info()['models'].keys())\n",
        "models"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zt0XYUgXm3Fb",
        "outputId": "8c472393-0562-4bdd-b4b5-10029e64f879"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['fasttext-wiki-news-subwords-300',\n",
              " 'conceptnet-numberbatch-17-06-300',\n",
              " 'word2vec-ruscorpora-300',\n",
              " 'word2vec-google-news-300',\n",
              " 'glove-wiki-gigaword-50',\n",
              " 'glove-wiki-gigaword-100',\n",
              " 'glove-wiki-gigaword-200',\n",
              " 'glove-wiki-gigaword-300',\n",
              " 'glove-twitter-25',\n",
              " 'glove-twitter-50',\n",
              " 'glove-twitter-100',\n",
              " 'glove-twitter-200',\n",
              " '__testing_word2vec-matrix-synopsis']"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tatoeba"
      ],
      "metadata": {
        "id": "d_6HopsNrq2T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tatoebatools"
      ],
      "metadata": {
        "id": "BVaa5yOTrioq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Monolingual corpus"
      ],
      "metadata": {
        "id": "fULo-lu6s7G6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tatoebatools import tatoeba"
      ],
      "metadata": {
        "id": "plU1pt_trwzF"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "en_df = tatoeba.get(\"sentences_detailed\", [\"eng\"])\n",
        "en_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "A702AnHxr_-M",
        "outputId": "63d8303a-cb3f-4e3c-d621-b769c5b84ef6"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "downloading https://downloads.tatoeba.org/exports/per_language/eng/eng_sentences_detailed.tsv.bz2\n",
            "100%|██████████| 25.4M/25.4M [00:01<00:00, 14.1MiB/s]\n",
            "decompressing eng_sentences_detailed.tsv.bz2\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         sentence_id lang                                               text  \\\n",
              "0               1276  eng                               Let's try something.   \n",
              "1               1277  eng                             I have to go to sleep.   \n",
              "2               1280  eng   Today is June 18th and it is Muiriel's birthday!   \n",
              "3               1282  eng                                 Muiriel is 20 now.   \n",
              "4               1283  eng                         The password is \"Muiriel\".   \n",
              "...              ...  ...                                                ...   \n",
              "1588747     10794497  eng  I think it would be better for us not to stay ...   \n",
              "1588748     10794500  eng                        You should've consulted me.   \n",
              "1588749     10794501  eng        He stood and waited for me for a long time.   \n",
              "1588750     10794502  eng                How much more time do we have left?   \n",
              "1588751     10794503  eng          I gained a lot of weight over the summer.   \n",
              "\n",
              "         username          date_added  date_last_modified  \n",
              "0              CK                 NaT 2012-02-05 11:38:18  \n",
              "1          vinhan                 NaT 2009-11-25 23:20:59  \n",
              "2          wuiwie                 NaT 2019-03-24 11:45:41  \n",
              "3        LeeSooHa                 NaT 2015-09-24 18:12:33  \n",
              "4          wuiwie                 NaT 2019-03-24 11:45:36  \n",
              "...           ...                 ...                 ...  \n",
              "1588747        CK 2022-04-30 05:14:10 2022-04-30 05:14:10  \n",
              "1588748        CK 2022-04-30 05:22:01 2022-04-30 05:22:01  \n",
              "1588749    ddnktr 2022-04-30 05:25:50 2022-04-30 05:25:50  \n",
              "1588750        CK 2022-04-30 05:26:42 2022-04-30 05:26:42  \n",
              "1588751        CK 2022-04-30 05:28:36 2022-04-30 05:28:36  \n",
              "\n",
              "[1588752 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b34d841b-63a1-4308-9c91-66e516503cf0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence_id</th>\n",
              "      <th>lang</th>\n",
              "      <th>text</th>\n",
              "      <th>username</th>\n",
              "      <th>date_added</th>\n",
              "      <th>date_last_modified</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1276</td>\n",
              "      <td>eng</td>\n",
              "      <td>Let's try something.</td>\n",
              "      <td>CK</td>\n",
              "      <td>NaT</td>\n",
              "      <td>2012-02-05 11:38:18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1277</td>\n",
              "      <td>eng</td>\n",
              "      <td>I have to go to sleep.</td>\n",
              "      <td>vinhan</td>\n",
              "      <td>NaT</td>\n",
              "      <td>2009-11-25 23:20:59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1280</td>\n",
              "      <td>eng</td>\n",
              "      <td>Today is June 18th and it is Muiriel's birthday!</td>\n",
              "      <td>wuiwie</td>\n",
              "      <td>NaT</td>\n",
              "      <td>2019-03-24 11:45:41</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1282</td>\n",
              "      <td>eng</td>\n",
              "      <td>Muiriel is 20 now.</td>\n",
              "      <td>LeeSooHa</td>\n",
              "      <td>NaT</td>\n",
              "      <td>2015-09-24 18:12:33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1283</td>\n",
              "      <td>eng</td>\n",
              "      <td>The password is \"Muiriel\".</td>\n",
              "      <td>wuiwie</td>\n",
              "      <td>NaT</td>\n",
              "      <td>2019-03-24 11:45:36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1588747</th>\n",
              "      <td>10794497</td>\n",
              "      <td>eng</td>\n",
              "      <td>I think it would be better for us not to stay ...</td>\n",
              "      <td>CK</td>\n",
              "      <td>2022-04-30 05:14:10</td>\n",
              "      <td>2022-04-30 05:14:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1588748</th>\n",
              "      <td>10794500</td>\n",
              "      <td>eng</td>\n",
              "      <td>You should've consulted me.</td>\n",
              "      <td>CK</td>\n",
              "      <td>2022-04-30 05:22:01</td>\n",
              "      <td>2022-04-30 05:22:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1588749</th>\n",
              "      <td>10794501</td>\n",
              "      <td>eng</td>\n",
              "      <td>He stood and waited for me for a long time.</td>\n",
              "      <td>ddnktr</td>\n",
              "      <td>2022-04-30 05:25:50</td>\n",
              "      <td>2022-04-30 05:25:50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1588750</th>\n",
              "      <td>10794502</td>\n",
              "      <td>eng</td>\n",
              "      <td>How much more time do we have left?</td>\n",
              "      <td>CK</td>\n",
              "      <td>2022-04-30 05:26:42</td>\n",
              "      <td>2022-04-30 05:26:42</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1588751</th>\n",
              "      <td>10794503</td>\n",
              "      <td>eng</td>\n",
              "      <td>I gained a lot of weight over the summer.</td>\n",
              "      <td>CK</td>\n",
              "      <td>2022-04-30 05:28:36</td>\n",
              "      <td>2022-04-30 05:28:36</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1588752 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b34d841b-63a1-4308-9c91-66e516503cf0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b34d841b-63a1-4308-9c91-66e516503cf0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b34d841b-63a1-4308-9c91-66e516503cf0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Parallel corpus"
      ],
      "metadata": {
        "id": "bza6JbjZs_tA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tatoebatools import ParallelCorpus"
      ],
      "metadata": {
        "id": "3l_qCC3-sz9E"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "zh2en = [(source.text, target.text) for source, target in ParallelCorpus(\"cmn\", \"eng\")]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rhWyjwkBuPmd",
        "outputId": "a6de02d1-3acd-4b3f-e972-7b4650150251"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "downloading https://downloads.tatoeba.org/exports/per_language/cmn/cmn-eng_links.tsv.bz2\n",
            "100%|██████████| 399k/399k [00:00<00:00, 990kiB/s] \n",
            "decompressing cmn-eng_links.tsv.bz2\n",
            "downloading https://downloads.tatoeba.org/exports/per_language/cmn/cmn_sentences_detailed.tsv.bz2\n",
            "100%|██████████| 1.33M/1.33M [00:00<00:00, 2.23MiB/s]\n",
            "decompressing cmn_sentences_detailed.tsv.bz2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "9q462yauu-1n"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "zh2en_df = pd.DataFrame(zh2en, columns =['Ch', 'En'])\n",
        "zh2en_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "I9iRIcbPuwbM",
        "outputId": "b24c1a12-9d4f-42fa-8d85-60390dfe1aa2"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   Ch                                 En\n",
              "0              我們試試看！                      Let's try it.\n",
              "1              我們試試看！                  Let's have a try.\n",
              "2              我們試試看！               Let's give it a try.\n",
              "3              我們試試看！                         Let's try!\n",
              "4              我們試試看！               Let's give it a try!\n",
              "...               ...                                ...\n",
              "65306         我能去你那吗？                   Can I come over?\n",
              "65307        她笑得停不下來。        She couldn't stop laughing.\n",
              "65308        她笑得停不下來。           She can't stop laughing.\n",
              "65309           哪兒來的？                   Where's it from?\n",
              "65310  不好意思，我不覺得我做得到。  Sorry, I don't think I can do it.\n",
              "\n",
              "[65311 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b91723e4-7ab4-4878-a1d5-6d4c18c5df2a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Ch</th>\n",
              "      <th>En</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>我們試試看！</td>\n",
              "      <td>Let's try it.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>我們試試看！</td>\n",
              "      <td>Let's have a try.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>我們試試看！</td>\n",
              "      <td>Let's give it a try.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>我們試試看！</td>\n",
              "      <td>Let's try!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>我們試試看！</td>\n",
              "      <td>Let's give it a try!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65306</th>\n",
              "      <td>我能去你那吗？</td>\n",
              "      <td>Can I come over?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65307</th>\n",
              "      <td>她笑得停不下來。</td>\n",
              "      <td>She couldn't stop laughing.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65308</th>\n",
              "      <td>她笑得停不下來。</td>\n",
              "      <td>She can't stop laughing.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65309</th>\n",
              "      <td>哪兒來的？</td>\n",
              "      <td>Where's it from?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65310</th>\n",
              "      <td>不好意思，我不覺得我做得到。</td>\n",
              "      <td>Sorry, I don't think I can do it.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>65311 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b91723e4-7ab4-4878-a1d5-6d4c18c5df2a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b91723e4-7ab4-4878-a1d5-6d4c18c5df2a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b91723e4-7ab4-4878-a1d5-6d4c18c5df2a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "en2fr = [(source.text, target.text) for source, target in ParallelCorpus(\"eng\", \"fra\")]\n",
        "en2fr_df = pd.DataFrame(en2fr, columns =['En', 'Fr'])\n",
        "en2fr_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 644
        },
        "id": "zLpimh5avmtY",
        "outputId": "38fd13cc-8bc0-4d36-8c9c-5a0287844628"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "downloading https://downloads.tatoeba.org/exports/per_language/eng/eng-fra_links.tsv.bz2\n",
            "100%|██████████| 1.88M/1.88M [00:00<00:00, 3.11MiB/s]\n",
            "decompressing eng-fra_links.tsv.bz2\n",
            "downloading https://downloads.tatoeba.org/exports/per_language/fra/fra_sentences_detailed.tsv.bz2\n",
            "100%|██████████| 10.2M/10.2M [00:00<00:00, 10.2MiB/s]\n",
            "decompressing fra_sentences_detailed.tsv.bz2\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                       En  \\\n",
              "0                                    Let's try something.   \n",
              "1                                    Let's try something.   \n",
              "2                                    Let's try something.   \n",
              "3                                  I have to go to sleep.   \n",
              "4                                    I have to go to bed.   \n",
              "...                                                   ...   \n",
              "319578       It's hard to be modest when you're the best.   \n",
              "319579  Tom said that he didn't know if he had time to...   \n",
              "319580                                   Did Tom like it?   \n",
              "319581  Tom and Mary say they can stay here for three ...   \n",
              "319582                                   Sami is running.   \n",
              "\n",
              "                                                       Fr  \n",
              "0                                Essayons quelque chose !  \n",
              "1                                 Tentons quelque chose !  \n",
              "2                                 Essayons quelque chose.  \n",
              "3                                   Je dois aller dormir.  \n",
              "4                                   Je dois aller dormir.  \n",
              "...                                                   ...  \n",
              "319578  C’est difficile d’être modeste quand on est le...  \n",
              "319579  Tom a dit qu’il ne savait pas s’il avait le te...  \n",
              "319580                         Est-ce que Tom a aimé ça ?  \n",
              "319581  Tom et Mary disent qu’ils peuvent rester ici t...  \n",
              "319582                                        Sami court.  \n",
              "\n",
              "[319583 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e66b0e99-da48-48ac-9ba6-4486c8475907\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>En</th>\n",
              "      <th>Fr</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Let's try something.</td>\n",
              "      <td>Essayons quelque chose !</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Let's try something.</td>\n",
              "      <td>Tentons quelque chose !</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Let's try something.</td>\n",
              "      <td>Essayons quelque chose.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I have to go to sleep.</td>\n",
              "      <td>Je dois aller dormir.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I have to go to bed.</td>\n",
              "      <td>Je dois aller dormir.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>319578</th>\n",
              "      <td>It's hard to be modest when you're the best.</td>\n",
              "      <td>C’est difficile d’être modeste quand on est le...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>319579</th>\n",
              "      <td>Tom said that he didn't know if he had time to...</td>\n",
              "      <td>Tom a dit qu’il ne savait pas s’il avait le te...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>319580</th>\n",
              "      <td>Did Tom like it?</td>\n",
              "      <td>Est-ce que Tom a aimé ça ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>319581</th>\n",
              "      <td>Tom and Mary say they can stay here for three ...</td>\n",
              "      <td>Tom et Mary disent qu’ils peuvent rester ici t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>319582</th>\n",
              "      <td>Sami is running.</td>\n",
              "      <td>Sami court.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>319583 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e66b0e99-da48-48ac-9ba6-4486c8475907')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e66b0e99-da48-48ac-9ba6-4486c8475907 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e66b0e99-da48-48ac-9ba6-4486c8475907');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# NLTK Corpus"
      ],
      "metadata": {
        "id": "Vqh-iEaCrO1O"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk"
      ],
      "metadata": {
        "id": "meM-AKKntoDh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Types of corpus structures\n",
        "- isolated: `gutenberg`\n",
        "- categorized: `brown`\n",
        "- overlapping: `reuters`\n",
        "- temporal: `inaugural`"
      ],
      "metadata": {
        "id": "xXfLijFPufLm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Brown"
      ],
      "metadata": {
        "id": "zrn0-Kaa8vMu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XN89re8YrMxO",
        "outputId": "c1bcbeab-4a85-4875-91c7-928cb5939c5f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<CategorizedTaggedCorpusReader in '.../corpora/brown' (not loaded yet)>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "nltk.corpus.brown"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('brown')\n",
        "from nltk.corpus import brown"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oo21tT0FvPyh",
        "outputId": "958c12e8-7ef5-43c4-f398-1397084b56aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/brown.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "brown.categories()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lXvosfbHuOgV",
        "outputId": "f9fa12b3-fc97-403d-d24d-605143a54c12"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['adventure',\n",
              " 'belles_lettres',\n",
              " 'editorial',\n",
              " 'fiction',\n",
              " 'government',\n",
              " 'hobbies',\n",
              " 'humor',\n",
              " 'learned',\n",
              " 'lore',\n",
              " 'mystery',\n",
              " 'news',\n",
              " 'religion',\n",
              " 'reviews',\n",
              " 'romance',\n",
              " 'science_fiction']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tagged_sents = brown.tagged_sents(categories='humor')\n",
        "len(tagged_sents)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zqf3IPTNvg2E",
        "outputId": "5d46d792-7e7d-4830-cb0f-59e25f713b93"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1053"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for sent in tagged_sents[:3]:\n",
        "    print(sent)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e2um72HRwY1L",
        "outputId": "b0c60624-fca1-4595-a4f0-728c866730e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('It', 'PPS'), ('was', 'BEDZ'), ('among', 'IN'), ('these', 'DTS'), ('that', 'CS'), ('Hinkle', 'NP'), ('identified', 'VBD'), ('a', 'AT'), ('photograph', 'NN'), ('of', 'IN'), ('Barco', 'NP'), ('!', '.'), ('!', '.')]\n",
            "[('For', 'CS'), ('it', 'PPS'), ('seems', 'VBZ'), ('that', 'CS'), ('Barco', 'NP'), (',', ','), ('fancying', 'VBG'), ('himself', 'PPL'), ('a', 'AT'), (\"ladies'\", 'NNS$'), ('man', 'NN'), ('(', '('), ('and', 'CC'), ('why', 'WRB'), ('not', '*'), (',', ','), ('after', 'IN'), ('seven', 'CD'), ('marriages', 'NNS'), ('?', '.'), ('?', '.')]\n",
            "[(')', ')'), (',', ','), ('had', 'HVD'), ('listed', 'VBN'), ('himself', 'PPL'), ('for', 'IN'), ('Mormon', 'NP-TL'), ('Beard', 'NN-TL'), ('roles', 'NNS'), ('at', 'IN'), ('the', 'AT'), ('instigation', 'NN'), ('of', 'IN'), ('his', 'PP$'), ('fourth', 'OD'), ('murder', 'NN'), ('victim', 'NN'), ('who', 'WPS'), ('had', 'HVD'), ('said', 'VBN'), (':', ':'), ('``', '``'), ('With', 'IN'), ('your', 'PP$'), ('beard', 'NN'), (',', ','), ('dear', 'JJ'), (',', ','), ('you', 'PPSS'), ('ought', 'MD'), ('to', 'TO'), ('be', 'BE'), ('in', 'IN'), ('movies', 'NNS'), (\"''\", \"''\"), ('!', '.'), ('!', '.')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Inaugural speeches"
      ],
      "metadata": {
        "id": "_oC_qp4s84eU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.corpus.inaugural"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UdB6m0-UzUyP",
        "outputId": "f35dde5f-35b5-4368-8958-282989c135fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<PlaintextCorpusReader in '.../corpora/inaugural' (not loaded yet)>"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('inaugural')\n",
        "from nltk.corpus import inaugural"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eim7V4KHxW44",
        "outputId": "b1999612-7179-43ef-9201-7f8a3f345edf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/inaugural.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "file_ids = inaugural.fileids()\n",
        "file_ids"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6OV3VN80xl1S",
        "outputId": "61aaf6ab-2f63-4230-e546-2c39c5579001"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['1789-Washington.txt',\n",
              " '1793-Washington.txt',\n",
              " '1797-Adams.txt',\n",
              " '1801-Jefferson.txt',\n",
              " '1805-Jefferson.txt',\n",
              " '1809-Madison.txt',\n",
              " '1813-Madison.txt',\n",
              " '1817-Monroe.txt',\n",
              " '1821-Monroe.txt',\n",
              " '1825-Adams.txt',\n",
              " '1829-Jackson.txt',\n",
              " '1833-Jackson.txt',\n",
              " '1837-VanBuren.txt',\n",
              " '1841-Harrison.txt',\n",
              " '1845-Polk.txt',\n",
              " '1849-Taylor.txt',\n",
              " '1853-Pierce.txt',\n",
              " '1857-Buchanan.txt',\n",
              " '1861-Lincoln.txt',\n",
              " '1865-Lincoln.txt',\n",
              " '1869-Grant.txt',\n",
              " '1873-Grant.txt',\n",
              " '1877-Hayes.txt',\n",
              " '1881-Garfield.txt',\n",
              " '1885-Cleveland.txt',\n",
              " '1889-Harrison.txt',\n",
              " '1893-Cleveland.txt',\n",
              " '1897-McKinley.txt',\n",
              " '1901-McKinley.txt',\n",
              " '1905-Roosevelt.txt',\n",
              " '1909-Taft.txt',\n",
              " '1913-Wilson.txt',\n",
              " '1917-Wilson.txt',\n",
              " '1921-Harding.txt',\n",
              " '1925-Coolidge.txt',\n",
              " '1929-Hoover.txt',\n",
              " '1933-Roosevelt.txt',\n",
              " '1937-Roosevelt.txt',\n",
              " '1941-Roosevelt.txt',\n",
              " '1945-Roosevelt.txt',\n",
              " '1949-Truman.txt',\n",
              " '1953-Eisenhower.txt',\n",
              " '1957-Eisenhower.txt',\n",
              " '1961-Kennedy.txt',\n",
              " '1965-Johnson.txt',\n",
              " '1969-Nixon.txt',\n",
              " '1973-Nixon.txt',\n",
              " '1977-Carter.txt',\n",
              " '1981-Reagan.txt',\n",
              " '1985-Reagan.txt',\n",
              " '1989-Bush.txt',\n",
              " '1993-Clinton.txt',\n",
              " '1997-Clinton.txt',\n",
              " '2001-Bush.txt',\n",
              " '2005-Bush.txt',\n",
              " '2009-Obama.txt',\n",
              " '2013-Obama.txt',\n",
              " '2017-Trump.txt',\n",
              " '2021-Biden.txt']"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "inaugural.raw(file_ids[-1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 249
        },
        "id": "URKSlqA67GnE",
        "outputId": "164f47b3-6a6b-4d81-9837-68e532144ff9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Chief Justice Roberts, Vice President Harris, Speaker Pelosi, Leader Schumer, Leader McConnell, Vice President Pence, and my distinguished guests, and my fellow Americans: This is America\\'s day. This is democracy\\'s day, a day of history and hope, of renewal and resolve. Through a crucible for the ages America has been tested anew, and America has risen to the challenge.\\n\\nToday we celebrate the triumph not of a candidate, but of a cause, the cause of democracy. The peopleâ\\x80\\x94the will of the people has been heard, and the will of the people has been heeded. We\\'ve learned again that democracy is precious, democracy is fragile. And at this hour, my friends, democracy has prevailed.\\n\\nSo now, on this hallowed ground where just a few days ago violence sought to shake the Capitol\\'s very foundation, we come together as one Nation under God, indivisible, to carry out the peaceful transfer of power as we have for more than two centuries. As we look ahead in our uniquely American wayâ\\x80\\x94restless, bold, optimisticâ\\x80\\x94and set our sights on the Nation we know we can be and we must be, I thank my predecessors of both parties for their presence here today. I thank them from the bottom of my heart. And I know the resilience of our Constitution and the strength, the strength of our Nation, as does President Carter, who I spoke with last night, who cannot be with us today, but whom we salute for his lifetime of service.\\n\\nI have just taken the sacred oath each of these patriots have taken, the oath first sworn by George Washington. But the American story depends not on any one of us, not on some of us, but on all of us, on \"We the People,\" who seek a more perfect Union.\\n\\nThis is a great Nation; we are good people. Over the centuries, through storm and strife, in peace and in war, we\\'ve come so far. But we still have far to go. We\\'ll press forward with speed and urgency, for we have much to do in this winter of peril and significant possibilities: much to repair, much to restore, much to heal, much to build, and much to gain.\\n\\nFew people in our Nation\\'s history have been more challenged or found a time more challenging or difficult than the time we\\'re in now. A once-in-a-century virus that silently stalks the country has taken as many lives in one year as America lost in all of World War II. Millions of jobs have been lost; hundreds of thousands of businesses closed. A cry for racial justice some 400 years in the making moves us. The dream of justice for all will be deferred no longer. A cry for survival comes from the planet itself, a cry that can\\'t be any more desperate or any more clear. And now, a rise of political extremism, White supremacy, domestic terrorism that we must confront and we will defeat.\\n\\nTo overcome these challengesâ\\x80\\x94to restore the soul and secure the future of Americaâ\\x80\\x94requires so much more than words. It requires the most elusive of all things in a democracy: unity. Unity.\\n\\nIn another January on New Year\\'s Day in 1863, Abraham Lincoln signed the Emancipation Proclamation. When he put pen to paper, the President said, and I quote, \"If my name ever goes down into history it will be for this act, and my whole soul is in it.\" My whole soul is in it.\\n\\nToday, on this January day, my whole soul is in this: bringing America together, uniting our people, uniting our Nation. And I ask every American to join me in this cause: uniting to fight the foes we face: anger, resentment, and hatred; extremism, lawlessness, violence; disease, joblessness, and hopelessness.\\n\\nWith unity we can do great things, important things. We can right wrongs. We can put people to work in good jobs. We can teach our children in safe schools. We can overcome the deadly virus. We can reward work and rebuild the middle class and make health care secure for all. We can deliver racial justice, and we can make America, once again, the leading force for good in the world.\\n\\nI know speaking of unity can sound to some like a foolish fantasy these days. I know the forces that divide us are deep and they are real. But I also know they are not new. Our history has been a constant struggle between the American ideal that we all are created equal and the harsh, ugly reality that racism, nativism, fear, demonization have long torn us apart. The battle is perennial. And victory is never assured.\\n\\nThrough the Civil War, the Great Depression, World War, 9/11, through struggle, sacrifice, and setbacks, our \"better angels\" have always prevailed. In each of these moments, enough of usâ\\x80\\x94enough of usâ\\x80\\x94have come together to carry all of us forward. And we can do that now. History, faith, and reason show the way: the way of unity.\\n\\nWe can see each other not as adversaries, but as neighbors. We can treat each other with dignity and respect. We can join forces, stop the shouting, and lower the temperature. For without unity, there is no peace, only bitterness and fury. No progress, only exhausting outrage. No nation, only a state of chaos.\\n\\nThis is our historic moment of crisis and challenge, and unity is the path forward. And we must meet this moment as the United States of America. If we do that, I guarantee you, we will not fail. We have never, ever, ever, ever failed in America when we have acted together.\\n\\nAnd so today, at this time, in this place, let\\'s start afresh, all of us. Let\\'s begin to listen to one another, hear one another, see one another, show respect to one another. Politics doesn\\'t have to be a raging fire destroying everything in its path. Every disagreement doesn\\'t have to be a cause for total war. And we must reject the culture in which facts themselves are manipulated and even manufactured.\\n\\nMy fellow Americans, we have to be different than this. America has to be better than this. And I believe America is so much better than this.\\n\\nJust look around. Here we stand, in the shadow of the Capitol dome, as was mentioned earlier, completed amid the Civil War, when the Union itself was literally hanging in the balance. Yet we endured, we prevailed.\\n\\nHere we stand looking out on the great Mall where Dr. King spoke of his dream.\\n\\nHere we stand, where 108 years ago, at another Inaugural, thousands of protesters tried to block brave women marching for the right to vote. And today we mark the swearing-in of the first woman in American history elected to national office: Vice President Kamala Harris. Don\\'t tell me things can\\'t change.\\n\\nHere we stand across the Potomac from Arlington Cemetery, where heroes who gave the last full measure of devotion rest in eternal peace.\\n\\nAnd here we stand, just days after a riotous mob thought they could use violence to silence the will of the people, to stop the work of our democracy, to drive us from this sacred ground. It did not happen. It will never happen, not today, not tomorrow, not everâ\\x80\\x94not ever.\\n\\nTo all those who supported our campaign, I am humbled by the faith you\\'ve placed in us. To all those who did not support us, let me say this: Hear me out as we move forward. Take a measure of me and my heart. And if you still disagree, so be it. That\\'s democracy. That\\'s America. The right to dissent peaceably, within the guardrails of our Republic, is perhaps this Nation\\'s greatest strength. Yet hear me clearly: Disagreement must not lead to disunion. And I pledge this to you: I will be a President for all Americansâ\\x80\\x94all Americans. And I promise you, I will fight as hard for those who did not support me as for those who did.\\n\\nMany centuries ago, Saint Augustine, a saint of my church, wrote that a people was a multitude defined by the common objects of their loveâ\\x80\\x94defined by the common objects of their love. What are the common objects we as Americans love? That define us as Americans? I think we know: opportunity, security, liberty, dignity, respect, honor, and yes, the truth.\\n\\nRecent weeks and months have taught us a painful lesson. There is truth, and there are lies: lies told for power and for profit. And each of us has a duty and a responsibility, as citizens, as Americans, and especially as leadersâ\\x80\\x94leaders who have pledged to honor our Constitution and protect our Nationâ\\x80\\x94to defend the truth and defeat the lies.\\n\\nLook, I understand that many of my fellow Americans view the future with fear and trepidation. I understand they worry about their jobs. I understand, like my dad, they lay at bed at night staring at the ceiling, wondering: \"Can I keep my health care? Can I pay my mortgage?\" Thinking about their families, about what comes next. I promise you, I get it. But the answer is not to turn inward, to retreat into competing factions, distrusting those who don\\'t look like you or worship the way you do or don\\'t get their news from the same sources you do. We must end this uncivil war that pits red against blue, rural versus urban, conservative versus liberal. We can do this if we open our souls instead of hardening our hearts, if we show a little tolerance and humility, and if we\\'re willing to stand in the other person\\'s shoes, as my mom would say, just for a moment. Stand in their shoes.\\n\\nBecause here is the thing about life: There is no accounting for what fate will deal you. Some days when you need a hand. There are other days when we\\'re called to lend a hand. That\\'s how it has to be. That is what we do for one another. And if we are this way, our country will be stronger, more prosperous, more ready for the future. And we can still disagree.\\n\\nMy fellow Americans, in the work ahead of us, we\\'re going to need each other. We need all our strength to persevere through this dark winter. We\\'re entering what may be the toughest and deadliest period of the virus. We must set aside politics and finally face this pandemic as one Nation. One Nation. And I promise you this: As the Bible says, \"Weeping may endure for a night, but joy cometh in the morning.\" We will get through this, togetherâ\\x80\\x94together.\\n\\nLook, folks, all my colleagues that I served with in the House and the Senate up here. We all understand the world is watching, watching all of us today. So here is my message to those beyond our borders: America has been tested, and we have come out stronger for it. We will repair our alliances and engage with the world once again, not to meet yesterday\\'s challenges, but today\\'s and tomorrow\\'s challenges. And we\\'ll lead not merely by the example of our power, but by the power of our example. We will be a strong and trusted partner for peace, progress, and security.\\n\\nLook, you all know, we have been through so much in this Nation. And, in my first act as President, I\\'d like to ask you to join me in a moment of silent prayer to remember all those we lost this past year to the pandemic, those 400,000 fellow Americans: moms, dads, husbands, wives, sons, daughters, friends, neighbors, and coworkers. We will honor them by becoming the people and Nation we know we can and should be.\\n\\nSo I ask you, let\\'s say a silent prayer for those who have lost their lives, for those they left behind, and for our country.\\n\\n[At this point, a moment of silence was observed.]\\n\\nAmen.\\n\\nFolks, this is a time of testing. We face an attack on our democracy and on truth, a raging virus, growing inequity, the sting of systemic racism, a climate in crisis, America\\'s role in the world. Any one of these would be enough to challenge us in profound ways. But the fact is, we face them all at once, presenting this Nation with one of the gravest of responsibilities we\\'ve had.\\n\\nNow we\\'re going to be tested. Are we going to step up? All of us? It is time for boldness, for there is so much to do. And this is certain, I promise you: We will be judged, you and I, by how we resolve these cascading crises of our era. Will we rise to the occasion, is the question. Will we master this rare and difficult hour? Will we meet our obligations and pass along a new and better world to our children? I believe we must; I\\'m sure you do as well. I believe we will.\\n\\nAnd when we do, we will write the next great chapter in the history of the United States of America: the American story, a story that might sound something like a song that means a lot to me. It\\'s called \"American Anthem,\" and there\\'s one verse that stands out, at least for me, and it goes like this:\\n\\n\"The work and prayers of centuries have brought us to this day\\nWhat shall be our legacy? What will our children say? . . .\\nLet me know in my heart when my days are through\\nAmerica, America, I gave my best to you.\"\\n\\nLet\\'s addâ\\x80\\x94let\\'s us add our own work and prayers to the unfolding story of our great Nation. If we do this, then when our days are through, our children and our children\\'s children will say of us: \"They gave their best. They did their duty. They healed a broken land.\"\\n\\nMy fellow Americans, I close today where I began, with a sacred oath. Before God and all of you I give you my word: I will always level with you. I will defend the Constitution. I will defend our democracy. I will defend America. And I will give all, all of you, keep everything youâ\\x80\\x94I do, in your service, thinking not of power, but of possibilities; not of personal interest, but of the public good. And together, we shall write an American story of hope, not fear; of unity, not division; of light, not darkness. A story of decency and dignity, love and healing, greatness and goodness.\\n\\nMay this be the story that guides us, the story that inspires us, and the story that tells ages yet to come that we answered the call of history, we met the moment; democracy and hope, truth and justice, did not die on our watch, but thrived; that America secured liberty at home and stood once again as a beacon to the world. That is what we owe our forebearers, one another, and generations to follow.\\n\\nSo with purpose and resolve we turn to those tasks of our time, sustained by faith, driven by conviction, and devoted to one another and the country we love with all our hearts.\\n\\nMay God bless America, and may God protect our troops. Thank you, America.\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "def inaugura2df(file_ids):\n",
        "    years = [id[:4] for id in file_ids]\n",
        "    presidents = [id[5:].replace(\".txt\", \"\") for id in file_ids]\n",
        "    texts = [inaugural.raw(id) for id in file_ids]\n",
        "    tokens = [inaugural.words(id) for id in file_ids]\n",
        "    df = pd.DataFrame(\n",
        "        {\n",
        "            \"year\": years,\n",
        "            \"president\": presidents,\n",
        "            \"text\": texts,\n",
        "            \"tokens\": tokens,\n",
        "        }\n",
        "    )\n",
        "    df.set_index(\"year\", inplace=True)\n",
        "    df.index = df.index.astype(int)\n",
        "    return df"
      ],
      "metadata": {
        "id": "H9FgFDhICN6M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "speeches = inaugura2df(file_ids)\n",
        "speeches"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "_EJu-T4dCxbJ",
        "outputId": "b2cbb56b-d646-4544-ee60-ece7c664e53e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       president                                               text  \\\n",
              "year                                                                  \n",
              "1789  Washington  Fellow-Citizens of the Senate and of the House...   \n",
              "1793  Washington  Fellow citizens, I am again called upon by the...   \n",
              "1797       Adams  When it was first perceived, in early times, t...   \n",
              "1801   Jefferson  Friends and Fellow Citizens:\\n\\nCalled upon to...   \n",
              "1805   Jefferson  Proceeding, fellow citizens, to that qualifica...   \n",
              "1809     Madison  Unwilling to depart from examples of the most ...   \n",
              "1813     Madison  About to add the solemnity of an oath to the o...   \n",
              "1817      Monroe  I should be destitute of feeling if I was not ...   \n",
              "1821      Monroe  Fellow citizens, I shall not attempt to descri...   \n",
              "1825       Adams  In compliance with an usage coeval with the ex...   \n",
              "1829     Jackson  Fellow citizens, about to undertake the arduou...   \n",
              "1833     Jackson  Fellow citizens, the will of the American peop...   \n",
              "1837    VanBuren  Fellow citizens: The practice of all my predec...   \n",
              "1841    Harrison  Called from a retirement which I had supposed ...   \n",
              "1845        Polk  Fellow citizens, without solicitation on my pa...   \n",
              "1849      Taylor  Elected by the American people to the highest ...   \n",
              "1853      Pierce  My Countrymen, It a relief to feel that no hea...   \n",
              "1857    Buchanan  Fellow citizens, I appear before you this day ...   \n",
              "1861     Lincoln  Fellow-Citizens of the United States: In compl...   \n",
              "1865     Lincoln  Fellow-Countrymen:\\n\\nAt this second appearing...   \n",
              "1869       Grant  Citizens of the United States:\\n\\nYour suffrag...   \n",
              "1873       Grant  Fellow-Citizens:\\n\\nUnder Providence I have be...   \n",
              "1877       Hayes  Fellow citizens, we have assembled to repeat t...   \n",
              "1881    Garfield  Fellow-Citizens:\\n\\nWe stand to-day upon an em...   \n",
              "1885   Cleveland  Fellow citizens, in the presence of this vast ...   \n",
              "1889    Harrison  Fellow-Citizens, there is no constitutional or...   \n",
              "1893   Cleveland  My Fellow citizens, in obedience of the mandat...   \n",
              "1897    McKinley  Fellow citizens, In obedience to the will of t...   \n",
              "1901    McKinley  My fellow-citizens, when we assembled here on ...   \n",
              "1905   Roosevelt  My fellow citizens, no people on earth have mo...   \n",
              "1909        Taft  My fellow citizens: Anyone who has taken the o...   \n",
              "1913      Wilson  There has been a change of government. It bega...   \n",
              "1917      Wilson  My Fellow citizens: The four years which have ...   \n",
              "1921     Harding  My Countrymen:\\n\\nWhen one surveys the world a...   \n",
              "1925    Coolidge  My countrymen,\\n\\nno one can contemplate curre...   \n",
              "1929      Hoover  My Countrymen: This occasion is not alone the ...   \n",
              "1933   Roosevelt  I am certain that my fellow Americans expect t...   \n",
              "1937   Roosevelt  When four years ago we met to inaugurate a Pre...   \n",
              "1941   Roosevelt  On each national day of inauguration since 178...   \n",
              "1945   Roosevelt  Chief Justice, Mr. Vice President, my friends,...   \n",
              "1949      Truman  Mr. Vice President, Mr. Chief Justice, and fel...   \n",
              "1953  Eisenhower  My friends, before I begin the expression of t...   \n",
              "1957  Eisenhower  The Price of Peace\\nMr. Chairman, Mr. Vice Pre...   \n",
              "1961     Kennedy  Vice President Johnson, Mr. Speaker, Mr. Chief...   \n",
              "1965     Johnson  My fellow countrymen, on this occasion, the oa...   \n",
              "1969       Nixon  Senator Dirksen, Mr. Chief Justice, Mr. Vice P...   \n",
              "1973       Nixon  Mr. Vice President, Mr. Speaker, Mr. Chief Jus...   \n",
              "1977      Carter  For myself and for our Nation, I want to thank...   \n",
              "1981      Reagan  Senator Hatfield, Mr. Chief Justice, Mr. Presi...   \n",
              "1985      Reagan  Senator Mathias, Chief Justice Burger, Vice Pr...   \n",
              "1989        Bush  Mr. Chief Justice, Mr. President, Vice Preside...   \n",
              "1993     Clinton  My fellow citizens, today we celebrate the mys...   \n",
              "1997     Clinton  My fellow citizens: At this last presidential ...   \n",
              "2001        Bush  President Clinton, distinguished guests and my...   \n",
              "2005        Bush  Vice President Cheney, Mr. Chief Justice, Pres...   \n",
              "2009       Obama  My fellow citizens:\\n\\nI stand here today humb...   \n",
              "2013       Obama  Thank you. Thank you so much.\\n\\nVice Presiden...   \n",
              "2017       Trump  Chief Justice Roberts, President Carter, Presi...   \n",
              "2021       Biden  Chief Justice Roberts, Vice President Harris, ...   \n",
              "\n",
              "                                                 tokens  \n",
              "year                                                     \n",
              "1789  (Fellow, -, Citizens, of, the, Senate, and, of...  \n",
              "1793  (Fellow, citizens, ,, I, am, again, called, up...  \n",
              "1797  (When, it, was, first, perceived, ,, in, early...  \n",
              "1801  (Friends, and, Fellow, Citizens, :, Called, up...  \n",
              "1805  (Proceeding, ,, fellow, citizens, ,, to, that,...  \n",
              "1809  (Unwilling, to, depart, from, examples, of, th...  \n",
              "1813  (About, to, add, the, solemnity, of, an, oath,...  \n",
              "1817  (I, should, be, destitute, of, feeling, if, I,...  \n",
              "1821  (Fellow, citizens, ,, I, shall, not, attempt, ...  \n",
              "1825  (In, compliance, with, an, usage, coeval, with...  \n",
              "1829  (Fellow, citizens, ,, about, to, undertake, th...  \n",
              "1833  (Fellow, citizens, ,, the, will, of, the, Amer...  \n",
              "1837  (Fellow, citizens, :, The, practice, of, all, ...  \n",
              "1841  (Called, from, a, retirement, which, I, had, s...  \n",
              "1845  (Fellow, citizens, ,, without, solicitation, o...  \n",
              "1849  (Elected, by, the, American, people, to, the, ...  \n",
              "1853  (My, Countrymen, ,, It, a, relief, to, feel, t...  \n",
              "1857  (Fellow, citizens, ,, I, appear, before, you, ...  \n",
              "1861  (Fellow, -, Citizens, of, the, United, States,...  \n",
              "1865  (Fellow, -, Countrymen, :, At, this, second, a...  \n",
              "1869  (Citizens, of, the, United, States, :, Your, s...  \n",
              "1873  (Fellow, -, Citizens, :, Under, Providence, I,...  \n",
              "1877  (Fellow, citizens, ,, we, have, assembled, to,...  \n",
              "1881  (Fellow, -, Citizens, :, We, stand, to, -, day...  \n",
              "1885  (Fellow, citizens, ,, in, the, presence, of, t...  \n",
              "1889  (Fellow, -, Citizens, ,, there, is, no, consti...  \n",
              "1893  (My, Fellow, citizens, ,, in, obedience, of, t...  \n",
              "1897  (Fellow, citizens, ,, In, obedience, to, the, ...  \n",
              "1901  (My, fellow, -, citizens, ,, when, we, assembl...  \n",
              "1905  (My, fellow, citizens, ,, no, people, on, eart...  \n",
              "1909  (My, fellow, citizens, :, Anyone, who, has, ta...  \n",
              "1913  (There, has, been, a, change, of, government, ...  \n",
              "1917  (My, Fellow, citizens, :, The, four, years, wh...  \n",
              "1921  (My, Countrymen, :, When, one, surveys, the, w...  \n",
              "1925  (My, countrymen, ,, no, one, can, contemplate,...  \n",
              "1929  (My, Countrymen, :, This, occasion, is, not, a...  \n",
              "1933  (I, am, certain, that, my, fellow, Americans, ...  \n",
              "1937  (When, four, years, ago, we, met, to, inaugura...  \n",
              "1941  (On, each, national, day, of, inauguration, si...  \n",
              "1945  (Chief, Justice, ,, Mr, ., Vice, President, ,,...  \n",
              "1949  (Mr, ., Vice, President, ,, Mr, ., Chief, Just...  \n",
              "1953  (My, friends, ,, before, I, begin, the, expres...  \n",
              "1957  (The, Price, of, Peace, Mr, ., Chairman, ,, Mr...  \n",
              "1961  (Vice, President, Johnson, ,, Mr, ., Speaker, ...  \n",
              "1965  (My, fellow, countrymen, ,, on, this, occasion...  \n",
              "1969  (Senator, Dirksen, ,, Mr, ., Chief, Justice, ,...  \n",
              "1973  (Mr, ., Vice, President, ,, Mr, ., Speaker, ,,...  \n",
              "1977  (For, myself, and, for, our, Nation, ,, I, wan...  \n",
              "1981  (Senator, Hatfield, ,, Mr, ., Chief, Justice, ...  \n",
              "1985  (Senator, Mathias, ,, Chief, Justice, Burger, ...  \n",
              "1989  (Mr, ., Chief, Justice, ,, Mr, ., President, ,...  \n",
              "1993  (My, fellow, citizens, ,, today, we, celebrate...  \n",
              "1997  (My, fellow, citizens, :, At, this, last, pres...  \n",
              "2001  (President, Clinton, ,, distinguished, guests,...  \n",
              "2005  (Vice, President, Cheney, ,, Mr, ., Chief, Jus...  \n",
              "2009  (My, fellow, citizens, :, I, stand, here, toda...  \n",
              "2013  (Thank, you, ., Thank, you, so, much, ., Vice,...  \n",
              "2017  (Chief, Justice, Roberts, ,, President, Carter...  \n",
              "2021  (Chief, Justice, Roberts, ,, Vice, President, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1f7283c9-f203-4e98-a98b-fbc474138337\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>president</th>\n",
              "      <th>text</th>\n",
              "      <th>tokens</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>year</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1789</th>\n",
              "      <td>Washington</td>\n",
              "      <td>Fellow-Citizens of the Senate and of the House...</td>\n",
              "      <td>(Fellow, -, Citizens, of, the, Senate, and, of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1793</th>\n",
              "      <td>Washington</td>\n",
              "      <td>Fellow citizens, I am again called upon by the...</td>\n",
              "      <td>(Fellow, citizens, ,, I, am, again, called, up...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1797</th>\n",
              "      <td>Adams</td>\n",
              "      <td>When it was first perceived, in early times, t...</td>\n",
              "      <td>(When, it, was, first, perceived, ,, in, early...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1801</th>\n",
              "      <td>Jefferson</td>\n",
              "      <td>Friends and Fellow Citizens:\\n\\nCalled upon to...</td>\n",
              "      <td>(Friends, and, Fellow, Citizens, :, Called, up...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1805</th>\n",
              "      <td>Jefferson</td>\n",
              "      <td>Proceeding, fellow citizens, to that qualifica...</td>\n",
              "      <td>(Proceeding, ,, fellow, citizens, ,, to, that,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1809</th>\n",
              "      <td>Madison</td>\n",
              "      <td>Unwilling to depart from examples of the most ...</td>\n",
              "      <td>(Unwilling, to, depart, from, examples, of, th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1813</th>\n",
              "      <td>Madison</td>\n",
              "      <td>About to add the solemnity of an oath to the o...</td>\n",
              "      <td>(About, to, add, the, solemnity, of, an, oath,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1817</th>\n",
              "      <td>Monroe</td>\n",
              "      <td>I should be destitute of feeling if I was not ...</td>\n",
              "      <td>(I, should, be, destitute, of, feeling, if, I,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1821</th>\n",
              "      <td>Monroe</td>\n",
              "      <td>Fellow citizens, I shall not attempt to descri...</td>\n",
              "      <td>(Fellow, citizens, ,, I, shall, not, attempt, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1825</th>\n",
              "      <td>Adams</td>\n",
              "      <td>In compliance with an usage coeval with the ex...</td>\n",
              "      <td>(In, compliance, with, an, usage, coeval, with...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1829</th>\n",
              "      <td>Jackson</td>\n",
              "      <td>Fellow citizens, about to undertake the arduou...</td>\n",
              "      <td>(Fellow, citizens, ,, about, to, undertake, th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1833</th>\n",
              "      <td>Jackson</td>\n",
              "      <td>Fellow citizens, the will of the American peop...</td>\n",
              "      <td>(Fellow, citizens, ,, the, will, of, the, Amer...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1837</th>\n",
              "      <td>VanBuren</td>\n",
              "      <td>Fellow citizens: The practice of all my predec...</td>\n",
              "      <td>(Fellow, citizens, :, The, practice, of, all, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1841</th>\n",
              "      <td>Harrison</td>\n",
              "      <td>Called from a retirement which I had supposed ...</td>\n",
              "      <td>(Called, from, a, retirement, which, I, had, s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1845</th>\n",
              "      <td>Polk</td>\n",
              "      <td>Fellow citizens, without solicitation on my pa...</td>\n",
              "      <td>(Fellow, citizens, ,, without, solicitation, o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1849</th>\n",
              "      <td>Taylor</td>\n",
              "      <td>Elected by the American people to the highest ...</td>\n",
              "      <td>(Elected, by, the, American, people, to, the, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1853</th>\n",
              "      <td>Pierce</td>\n",
              "      <td>My Countrymen, It a relief to feel that no hea...</td>\n",
              "      <td>(My, Countrymen, ,, It, a, relief, to, feel, t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1857</th>\n",
              "      <td>Buchanan</td>\n",
              "      <td>Fellow citizens, I appear before you this day ...</td>\n",
              "      <td>(Fellow, citizens, ,, I, appear, before, you, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1861</th>\n",
              "      <td>Lincoln</td>\n",
              "      <td>Fellow-Citizens of the United States: In compl...</td>\n",
              "      <td>(Fellow, -, Citizens, of, the, United, States,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1865</th>\n",
              "      <td>Lincoln</td>\n",
              "      <td>Fellow-Countrymen:\\n\\nAt this second appearing...</td>\n",
              "      <td>(Fellow, -, Countrymen, :, At, this, second, a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1869</th>\n",
              "      <td>Grant</td>\n",
              "      <td>Citizens of the United States:\\n\\nYour suffrag...</td>\n",
              "      <td>(Citizens, of, the, United, States, :, Your, s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1873</th>\n",
              "      <td>Grant</td>\n",
              "      <td>Fellow-Citizens:\\n\\nUnder Providence I have be...</td>\n",
              "      <td>(Fellow, -, Citizens, :, Under, Providence, I,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1877</th>\n",
              "      <td>Hayes</td>\n",
              "      <td>Fellow citizens, we have assembled to repeat t...</td>\n",
              "      <td>(Fellow, citizens, ,, we, have, assembled, to,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1881</th>\n",
              "      <td>Garfield</td>\n",
              "      <td>Fellow-Citizens:\\n\\nWe stand to-day upon an em...</td>\n",
              "      <td>(Fellow, -, Citizens, :, We, stand, to, -, day...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1885</th>\n",
              "      <td>Cleveland</td>\n",
              "      <td>Fellow citizens, in the presence of this vast ...</td>\n",
              "      <td>(Fellow, citizens, ,, in, the, presence, of, t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1889</th>\n",
              "      <td>Harrison</td>\n",
              "      <td>Fellow-Citizens, there is no constitutional or...</td>\n",
              "      <td>(Fellow, -, Citizens, ,, there, is, no, consti...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1893</th>\n",
              "      <td>Cleveland</td>\n",
              "      <td>My Fellow citizens, in obedience of the mandat...</td>\n",
              "      <td>(My, Fellow, citizens, ,, in, obedience, of, t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1897</th>\n",
              "      <td>McKinley</td>\n",
              "      <td>Fellow citizens, In obedience to the will of t...</td>\n",
              "      <td>(Fellow, citizens, ,, In, obedience, to, the, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1901</th>\n",
              "      <td>McKinley</td>\n",
              "      <td>My fellow-citizens, when we assembled here on ...</td>\n",
              "      <td>(My, fellow, -, citizens, ,, when, we, assembl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1905</th>\n",
              "      <td>Roosevelt</td>\n",
              "      <td>My fellow citizens, no people on earth have mo...</td>\n",
              "      <td>(My, fellow, citizens, ,, no, people, on, eart...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1909</th>\n",
              "      <td>Taft</td>\n",
              "      <td>My fellow citizens: Anyone who has taken the o...</td>\n",
              "      <td>(My, fellow, citizens, :, Anyone, who, has, ta...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1913</th>\n",
              "      <td>Wilson</td>\n",
              "      <td>There has been a change of government. It bega...</td>\n",
              "      <td>(There, has, been, a, change, of, government, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1917</th>\n",
              "      <td>Wilson</td>\n",
              "      <td>My Fellow citizens: The four years which have ...</td>\n",
              "      <td>(My, Fellow, citizens, :, The, four, years, wh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1921</th>\n",
              "      <td>Harding</td>\n",
              "      <td>My Countrymen:\\n\\nWhen one surveys the world a...</td>\n",
              "      <td>(My, Countrymen, :, When, one, surveys, the, w...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1925</th>\n",
              "      <td>Coolidge</td>\n",
              "      <td>My countrymen,\\n\\nno one can contemplate curre...</td>\n",
              "      <td>(My, countrymen, ,, no, one, can, contemplate,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1929</th>\n",
              "      <td>Hoover</td>\n",
              "      <td>My Countrymen: This occasion is not alone the ...</td>\n",
              "      <td>(My, Countrymen, :, This, occasion, is, not, a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1933</th>\n",
              "      <td>Roosevelt</td>\n",
              "      <td>I am certain that my fellow Americans expect t...</td>\n",
              "      <td>(I, am, certain, that, my, fellow, Americans, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1937</th>\n",
              "      <td>Roosevelt</td>\n",
              "      <td>When four years ago we met to inaugurate a Pre...</td>\n",
              "      <td>(When, four, years, ago, we, met, to, inaugura...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1941</th>\n",
              "      <td>Roosevelt</td>\n",
              "      <td>On each national day of inauguration since 178...</td>\n",
              "      <td>(On, each, national, day, of, inauguration, si...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1945</th>\n",
              "      <td>Roosevelt</td>\n",
              "      <td>Chief Justice, Mr. Vice President, my friends,...</td>\n",
              "      <td>(Chief, Justice, ,, Mr, ., Vice, President, ,,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1949</th>\n",
              "      <td>Truman</td>\n",
              "      <td>Mr. Vice President, Mr. Chief Justice, and fel...</td>\n",
              "      <td>(Mr, ., Vice, President, ,, Mr, ., Chief, Just...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1953</th>\n",
              "      <td>Eisenhower</td>\n",
              "      <td>My friends, before I begin the expression of t...</td>\n",
              "      <td>(My, friends, ,, before, I, begin, the, expres...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1957</th>\n",
              "      <td>Eisenhower</td>\n",
              "      <td>The Price of Peace\\nMr. Chairman, Mr. Vice Pre...</td>\n",
              "      <td>(The, Price, of, Peace, Mr, ., Chairman, ,, Mr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1961</th>\n",
              "      <td>Kennedy</td>\n",
              "      <td>Vice President Johnson, Mr. Speaker, Mr. Chief...</td>\n",
              "      <td>(Vice, President, Johnson, ,, Mr, ., Speaker, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1965</th>\n",
              "      <td>Johnson</td>\n",
              "      <td>My fellow countrymen, on this occasion, the oa...</td>\n",
              "      <td>(My, fellow, countrymen, ,, on, this, occasion...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1969</th>\n",
              "      <td>Nixon</td>\n",
              "      <td>Senator Dirksen, Mr. Chief Justice, Mr. Vice P...</td>\n",
              "      <td>(Senator, Dirksen, ,, Mr, ., Chief, Justice, ,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1973</th>\n",
              "      <td>Nixon</td>\n",
              "      <td>Mr. Vice President, Mr. Speaker, Mr. Chief Jus...</td>\n",
              "      <td>(Mr, ., Vice, President, ,, Mr, ., Speaker, ,,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1977</th>\n",
              "      <td>Carter</td>\n",
              "      <td>For myself and for our Nation, I want to thank...</td>\n",
              "      <td>(For, myself, and, for, our, Nation, ,, I, wan...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1981</th>\n",
              "      <td>Reagan</td>\n",
              "      <td>Senator Hatfield, Mr. Chief Justice, Mr. Presi...</td>\n",
              "      <td>(Senator, Hatfield, ,, Mr, ., Chief, Justice, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1985</th>\n",
              "      <td>Reagan</td>\n",
              "      <td>Senator Mathias, Chief Justice Burger, Vice Pr...</td>\n",
              "      <td>(Senator, Mathias, ,, Chief, Justice, Burger, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1989</th>\n",
              "      <td>Bush</td>\n",
              "      <td>Mr. Chief Justice, Mr. President, Vice Preside...</td>\n",
              "      <td>(Mr, ., Chief, Justice, ,, Mr, ., President, ,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1993</th>\n",
              "      <td>Clinton</td>\n",
              "      <td>My fellow citizens, today we celebrate the mys...</td>\n",
              "      <td>(My, fellow, citizens, ,, today, we, celebrate...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1997</th>\n",
              "      <td>Clinton</td>\n",
              "      <td>My fellow citizens: At this last presidential ...</td>\n",
              "      <td>(My, fellow, citizens, :, At, this, last, pres...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2001</th>\n",
              "      <td>Bush</td>\n",
              "      <td>President Clinton, distinguished guests and my...</td>\n",
              "      <td>(President, Clinton, ,, distinguished, guests,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2005</th>\n",
              "      <td>Bush</td>\n",
              "      <td>Vice President Cheney, Mr. Chief Justice, Pres...</td>\n",
              "      <td>(Vice, President, Cheney, ,, Mr, ., Chief, Jus...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009</th>\n",
              "      <td>Obama</td>\n",
              "      <td>My fellow citizens:\\n\\nI stand here today humb...</td>\n",
              "      <td>(My, fellow, citizens, :, I, stand, here, toda...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2013</th>\n",
              "      <td>Obama</td>\n",
              "      <td>Thank you. Thank you so much.\\n\\nVice Presiden...</td>\n",
              "      <td>(Thank, you, ., Thank, you, so, much, ., Vice,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2017</th>\n",
              "      <td>Trump</td>\n",
              "      <td>Chief Justice Roberts, President Carter, Presi...</td>\n",
              "      <td>(Chief, Justice, Roberts, ,, President, Carter...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2021</th>\n",
              "      <td>Biden</td>\n",
              "      <td>Chief Justice Roberts, Vice President Harris, ...</td>\n",
              "      <td>(Chief, Justice, Roberts, ,, Vice, President, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1f7283c9-f203-4e98-a98b-fbc474138337')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1f7283c9-f203-4e98-a98b-fbc474138337 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1f7283c9-f203-4e98-a98b-fbc474138337');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cfd = nltk.ConditionalFreqDist(\n",
        "                (target, fileid[:4])\n",
        "                for fileid in inaugural.fileids()\n",
        "                for w in inaugural.words(fileid)\n",
        "                for target in ['asia', 'europe']\n",
        "                if w.lower().startswith(target))\n",
        "cfd.plot()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "Z1jO_no_0S0F",
        "outputId": "eb458db9-c63f-46b8-8180-83f77e75b190"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEWCAYAAABsY4yMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZgU1dX/P2d2BmaGfd+3YQedEXBBERFRcEs0ifnFaBZNXhPjG2M0iYAbLknUmBh9s2lMYmISTVxAEUHBDUEYAdl32bcZYGCYffr8/qiaoRlnoJeqruru+3mefqamu+t7T3V3nb5969zvFVXFYDAYDIlHitcBGAwGg8EdTII3GAyGBMUkeIPBYEhQTII3GAyGBMUkeIPBYEhQTII3GAyGBCXN6wCCad++vfbu3TuifSsqKmjRooVjsfhdzw1Nv+u5oZlsem5o+l3PDU0/6RUVFRWraocmH1RV39wKCgo0UpYtWxbxvvGo54am3/Xc0Ew2PTc0/a7nhqaf9IBl2kxONUM0BoPBkKCYBG8wGAwJiknwBoPBkKD46iKrwWAwANTU1LBr1y4qKysd0UtLS2PdunWOaHmll5WVRffu3UlPTw9dN9rAToWItAb+BAwDFPimqn7kZpsGgyH+2bVrFzk5OfTu3RsRiVrv+PHjtGzZ0oHIvNFTVUpKSti1axd9+vQJWdftIZpfA2+q6iBgJODcV57BYEhYKisradeunSPJPREQEdq1axf2LxrXEryI5AHnA88AqGq1qh5xqz2DIeGoqyHz+G6vo/AMk9xPJpLXw80efB/gIPBnEVkuIn8SEed+0xgMic6iJxn2zvWw9lWvIzGcghkzZjB//nyvw2gSUZcW/BCRQmAxcK6qLhGRXwNHVXV6o+fdDNwM0KVLl4JZs2ZF1F55eTnZ2dlRRh0/em5o+l3PDU0/6/X7eBqt9y/iSKdz2DJ6piOa4O9jrtfLzc2lf//+jmkGAgFSUpzrz3qlt3nzZmpra0+6r7CwsEhVC5vcobkZUNHegM7AZ0H/jwNeP9U+Ziart5p+13ND09d6T5+jek+u6gOdVKuOOybr62O29dauXeuoZllZWdj7XHnllXrmmWfqkCFD9Pe//73W1tbqDTfcoEOHDtUhQ4bo448/rqqqN9xwg7744ouqqnrfffdpYWGhDh06VG+66SYNBAKOxtfU68IpZrK6VkWjqvtEZKeI5KvqBuAiYK1b7RkMCceRHdbf2grYuhAGXeZpOF7R+yevu6L72SNTTvn4s88+S9u2bamoqOCss86ioKCA3bt3s3r1ao4fP05NTc3n9vn+97/PjBkzALj++uuZPXs2l19+uSvxh4LbVTS3An8XkU+BUcBDLrdnMCQGFUeg6uiJ/ze4k+QMzfOb3/yGkSNHMnbsWHbu3El1dTVbt27l1ltvZd68eeTm5n5unwULFjBmzBiGDx/OO++8w5o1azyI/ASu1sGr6gqg6bEhg8HQPHbvvS61Bal1FbDhTQjUQUqqx4HFntP1tEMh3Lr1hQsXMn/+fD766COys7MZP348VVVVrFy5krlz5/LMM8/w2muv8eyzzzbsU1lZyS233MKyZcvo0aMH9957r2MTtSLFWBUYDH6kdCcAZe2GQ+teUF4Mu5Z6HFTyUFpaSps2bcjOzmb9+vUsXryY4uJiAoEAX/ziF5k+fTqffPLJSfvUJ/P27dtTVlbGSy+95EXoJ2GsCgwGP2L34KtadIa+hbD4aVj/OvQc63FgycHkyZP53e9+x+DBg8nPz2fs2LHs3r2b8ePHEwgECAQC/PznPz9pn9atW3PTTTcxbNgwOnfuzFlnneVR9CcwCd5g8CNHrB58dXZnyL/MSvAb3oBJD3gcWHKQmZnJnDlzPnf/bbfdBpw85PPcc881PD5z5kxmznSupDVazBCNweBHjmwHoLpFJ+h5NrRoAyWboXiTx4EZ4gmT4A0GP2KPwVdld4bUNBhwiXX/elNNYwgdk+ANBj9ij8FXt+hk/Z9/qfV3wxseBWSIR0yCNxj8RtUxqDgMaVnUZrax7ut/EaRmwM6PoeyAt/EZ4gaT4A0Gv2FfYCWvO9Q7CGbmQJ8LAIWNb3oWmiG+MAneYPAb9vg7rXuefH+9VcF6M0xjCA2T4A0Gv1HvQZPX4+T7B9rj8FsXQHV5bGMyxCUmwRsMfqM+wTfuwed2gW4FUFtpJXmDL2ls5+slJsEbDH6juQQPJ6ppzDCN6zz//POMHj2aUaNG8Z3vfIe6ujpatWrV8PhLL73EjTfeCMCNN97Id7/7XcaMGcOdd97JihUrGDt2LCNGjODqq6/m8OHDAIwfP57bbruNUaNGMWzYMD7++GPAmjj1zW9+k9GjR3PGGWfw6qvOLPJiZrIaDH4jOMEfbPRY/hR4Z6Z1oTVZzMfuzYtaokmbsXtLm33+unXr+Ne//sWHH35Ieno6t9xyC3//+99P2cauXbtYtGgRqampjBgxgieffJILLriAGTNmcN999/HEE08A1oImK1as4L333uOb3/wmq1ev5pe//CUTJkzg2Wef5ciRI4wePZqJEydGvbC3SfAGg9+ov8ia1wMO7jv5sY6DoU1vOPyZVTLZ6+xYR5cUvP322xQVFTX4yVRUVNCxY8dT7nPttdeSmppKaWkpR44c4YILLgDghhtu4Nprr2143nXXXQfA+eefz9GjRzly5Ahvv/02c+bM4dFHHwUs47IdO3YwePDgqI7DJHiDwU9Ul8Pxg5CSDjmdgUYJXsTqxS9+yvKIT4YEf4qedqiEaxesqtxwww08/PDDJ93/2GOPNWw3tgIOVb/x4tkigqryn//8h/z8/JBjDAUzBm8w+InSXdbfvO7ND7/Ul0tu+LwZlsEZLrroIl566SUOHLAmlR06dIjt27fTqVMn1q1bRyAQ4OWXX25y37y8PNq0acP7778PwN/+9reG3jzAv/71LwA++OAD8vLyyMvLY+LEiTz55JP1y5uyfPlyR47D9OANBj/RMP7eo/nn9Bh7wnzs4EboMDA2sSURQ4YMYebMmUyaNIlAIEB6ejpPPfUUjzzyCFOnTqVdu3aMHj2asrKyJvf/y1/+wne/+13Ky8vp27cvf/7znxsey8rK4owzzqCmpqZhwZC77rqLu+++mxEjRhAIBOjTpw+zZ8+O+jhMgjcY/ETpKSpo6qk3H/v0n9YwjUnwrvDlL3+ZL3/5y5+7/5prrvnckE+wZTDAqFGjWLx4cZO6X/va1xouuNbTokULfv/730cfdCPMEI3B4CcaJjmdIsGDmdVqCAnTgzcY/MSRZmwKGtPvIkjNtJbxKzsArU5d4WHwBwsXLoxpe6YHbzD4iVDG4AEyW0Ff23zMXGw1NINJ8AaDn2jOaKwp8hO7mqa+osRgEcnrYRK8weAXaqvg2F6QVMjpevrn5webjx13N7YYk5WVRUlJiUnyNqpKSUkJWVlZYe1nxuANBr9QXwOf282qlDkdOZ0t87HdRbBlAQye6m58MaR79+7s2rWLgwcbezVERnV1NRkZGY5oeaWXlZVF9+7dw9J1NcGLyGfAMaAOqFXVQjfbMxjimlDH34PJv8xK8BveSKgEn56eTp8+fRzTKyoqYuTIkUmjV08shmguVNVRJrkbDKchnPH3egZNsf7Wm48ZDEGYMXiDwS80t9DHqegwCNr0gfIS2LnEnbgMcYvbCV6Bt0SkSERudrktgyG+OZUPfHOInOjFr3/d+ZgMcY24eZVaRLqp6m4R6QjMA25V1fcaPedm4GaALl26FMyaNSuitsrLy8nOzo425LjRc0PT73puaPpJb+CHt5FzaBUbxz7KsQ5nhqzXqmQl+Yt+SGXLbqy58K8nFup2IcZ41HND0096hYWFRc0OgatqTG7AvcAdp3pOQUGBRsqyZcsi3jce9dzQ9LueG5q+0ntsiOo9uaolW8LTq61RfaSXte+B9e7GGId6bmj6SQ9Yps3kVNeGaESkpYjk1G8Dk4DVbrVnMMQ1dTVwbA8gkBteKRypaTBwsrVthmkMQbg5Bt8J+EBEVgIfA6+r6psutmcwxC9Hd4MGIKcLpEVQX90wq9WYjxlO4FodvKpuBZwv7DQYEpFQTcaao98E23xsGRzbDzmdnIvNELeYMkmDwQ9EMskpmMxW0Hc8oLAxMb1pDOFjErzB4AcimeTUGLOUn6ERJsEbDH4gkklOjam/0Lp1YcKZjxkiwyR4g8EPRDLJqTE5naFbIdRWwpZ3nInLENeYBG8w+AEnEjyYpfwMJ2ESvMHgNYE6q0wSIC/MGvjG5AeZj9XVRqdliHtMgjcYvObYXgjUQsuOkN4iOq0O+dC2L1QcMuZjBpPgDQbPcWp4BiwfGjPpyWBjErzB4DUNk5yiqKAJJjjBmyXvkhqT4A0Gr3GyBw/QYwy0aAuHtsLBDc5oGuISk+ANBq8pdaAGPphg87ENxnwsmTEJ3mDwmoYefC/nNE25pAGT4A0G74nWaKwp+k2AtCzYvQyO7XNO1xBXmARvMHhJIBDkQ+PQEA1ARkvbfAyrJt6QlJgEbzB4yfEDUFcN2e2spOwk+Zdaf80wTdJiErzB4CVOmIw1x8BLAbHMx6rKnNc3+B6T4A0GL3G6RDKYnE7QvRDqqoz5WJJiErzB4CVuJngws1qTHJPgDQYvcWKhj1MxyJiPJTMmwRsMXuLmGDxA+4HQth9UHDbmY0mISfAGg5e4UQMfjMiJahozTJN0mARvMHiFavSLbYdC/TDN+teN+ViSYRK8weAVx4uhtgKy8qybW/QYY9XZH94GB9e7147Bd5gEbzB4RYPJmEvDM/WkpJ4wH1tvzMeSCdcTvIikishyEZntdlsGQ1zhdolkMKZcMimJRQ/+NmBdDNoxGOILpxf6OBX9LrTNx4pIqyxxvz2DL3A1wYtId2AK8Cc32zEY4pJY9uCDzMda7//I/fYMvsDtHvwTwJ1AwOV2DIb4w+1JTo2xh2nab58NgbrYtGnwFFGXyqZEZCpwmareIiLjgTtUdWoTz7sZuBmgS5cuBbNmzYqovfLycrKzs6OIOL703ND0u54bml7qDVn4LVoc28bacb+jovVA1+NLqS1n6IIbyagsZvvw2yjufaUjun5/T9zQ9JNeYWFhkaoWNvmgqrpyAx4GdgGfAfuAcuD5U+1TUFCgkbJs2bKI941HPTc0/a7nhqZneoGA6oPdVO/JVT1eEr1eqKx5xWrzoR6qx/Y7Iun398QNTT/pAcu0mZzq2hCNqv5UVburam/gK8A7qvo1t9ozGOKKisNQfQwyWkGLNrFrd/AVlHYcDVWlMPfu2LVr8ARTB28weEHw+LtI7NoVYcewH1gVNav+bXnFGxKWmCR4VV2oTYy/GwxJi9smY6egumVXOP/H1j+v/whqq2IegyE2mB68weAFbpuMnY5zfmA5TZZshg9/7U0MBtcxCd5g8IJYmIydirQMmPK4tf3eo1CyxZs4DK5iErzB4AWxroFvij7jYMRXrCX93rjDOE0mICbBGwxecGS79ddto7HTMWmm5WS55R1Y87K3sRgcxyR4g8ELYmlTcCpadYCJ91nbb/4UKo96G4/BUUyCNxhiTWWpdUtrAS3bex0NnHkDdD8LyvbBgge9jsbgICbBGwyxJthFMpY18M2RkgJTfwWSCh//AfYs9zoig0OYBG8wxJr6C6we1MA3S+fhMPZ/QAMw+4fGjCxBMAneYIg1fhl/b8z4n0BOV6sHv+xZr6MxOIBJ8AZDrPFrgs/MgUt/bm2/fT8c2+9tPIaoMQneYIg1fk3wAIMvhwGXQNVRmPszr6MxRIlJ8AZDrPHDJKfmEIHLfmFV+Kx+CbYs8DoiQxSEneBFpI2IjHAjGIMhKfDQaCwk2vSGC4LMyGoqPQ3HEDkhJXgRWSgiuSLSFvgE+KOIPO5uaAZDAlJ9HMpLIDUDWnXyOprmOftWaJ8Ph7bAh094HY0hQkLtweep6lHgC8BfVXUMMNG9sAyGBKW+Bj6vu1V/7lfSMmCq3Yd7/3FjRhanhPoJSxORLsCXgNkuxmMwJDZ+Hn9vTO/zYORXLTOy139kzMjikFAT/H3AXGCzqi4Vkb7AJvfCMhgSlAaTMZ+Ovzdm0gOQ1Rq2LoDV//E6GkOYhJrg96rqCFW9BUBVtwJmDN5gCJcGm4Je3sYRKi3bw8W2Gdncn1keOoa4IdQE/2SI9xkMhlPh9UIfkXDG16H7aCjbD+/M9DoaQxiknepBETkbOAfoICK3Bz2UC6S6GZjBkJD4eZJTc9Sbkf3+fPj4jzDyOuh2ptdRGULgdD34DKAV1hdBTtDtKHCNu6EZDAmIH43GQqHzMMuMDDVmZHHEKXvwqvou8K6IPKeq22MUk8GQmNRUWsMcKWmQ08XraMJn/E+tVZ/2roClz8CYm72OyHAaQh2DzxSRP4jIWyLyTv3N1cgMhkSjdJf1N7crpJ6yb+VPMlvBpb+wtt95AI7t8zYew2kJ9VP2IvA74E+A+W1mMERCfYlkvFTQNMWgKTBwMmx806qqucbYCvuZUBN8rar+XzjCIpIFvAdk2u28pKr3hBmfwZA4xNMkp+YQsXrxW9+16uJH/T+gtddRGZoh1CGaWSJyi4h0EZG29bfT7FMFTFDVkcAoYLKIjI0qWoMhnvG7yViotOkF4++ytt+4A6mr9jYeQ7OE2oO/wf7746D7FOjb3A6qqkCZ/W+6fTNznZ0gEICNb5JWlel1JIZwOJIAPfh6xn4PVv4TDq6n18pHQS9xTLrDzh1QW+SYHmkZpNb0cU4vjggpwatqRK+OiKQCRUB/4ClVXRKJjqERi34N8++lZ5fz4ZyLvI7GECrxOMmpOdIyYMrj8NxltNs9H3bPd0y6J8Bqx+QA6Nr7Shh7gbOicYBoCAZCIvL1pu5X1b+G1IhIa+Bl4FZVXd3osZuBmwG6dOlSMGvWrFAkP0d5eTnZ2dkR7RtPehnl+xi64BukBKqoS8lk5SUvo2lZDkTo32N2UzOWesPnfZmMyoOsuujvVGeHVibp99ew7c63yCxeRVqac1VBtbW1juml1FXSfudcqjPasGrSiyDOOHj66XNYWFhYpKqFTT6oqqe9YdkS1N/+CGzFumga0v62xgzgjlM9p6CgQCNl2bJlEe8bN3qBgOrfv6R6T+6J27rXo9e18eUxu6wZM72aKtV78lTvba1aWx29XhTE7WsYCYGA6uNDrXNl51LHZP10zMAybSanhvR1pqq3Bt1uAs7EmuHaLCLSwe65IyItgIuB9aG0Z2iG9a9b5WmZuXDWt637NrzhbUyG0Di6G1DI6Qqp6V5HkzyIQP6l1nYSniuR/l45DpxuXL4LsEBEPgWWAvNU1XjJR0pVGcyxKxcmTIPCb1rbG98008bjgUQaf4836hP8+uRL8CENdInILE5UwKQCg4F/n2ofVf0UOCOq6AwnePcROLoLuoyyeu+SQlWLzmQe3we7lkHPMV5HaDgV8Wgylij0Oo+6tJakHlwHh7ZC22aL/xKOUHvwjwKP2beHgPNV9SeuRWU4mX2r4aOnAbFc/VJSQYQjnc+xHt/wuqfhGUIgXk3GEoG0DEo7jra2k6wXH+oY/LtY4+c5QBvAzGyIFYEAvH47aJ3Vcw+yaT3S+VxrI8k+tHGJ6cF7SsO5kmTj8CEleBH5EvAxcC3WuqxLRMTYBceC5X+DnUugVSe4aPpJD5W1HWEtp1ayCYrNCoq+pmGSk+nBe0Fpx9GQkg47PoLjJV6HEzNCHaK5GzhLVW9Q1a8Do4Hpp9nHEC3Hi2HeDGv7kocgK+/kx1NSYaA9gzDJeiZxR0MPPo6NxuKYQHoraxFxDcCmt7wOJ2aEmuBTVPVA0P8lYexriJR5M6DyCPQdD8O+2PRz8i+z/pphGv9SV2uXSQK53byNJZmpP1eS6JpVqEn6TRGZKyI3isiNwOuAyShu8tkHsOLvkJppTQkXafp5/S+C1AxrGKfsYGxjNITGsT3WNZRWnSHdmVnHhgioL5fc/I61+EoScMoELyL9ReRcVf0x8HtghH37CPhDDOJLTmqrYba9BO6426Fdv+afm5kDfc4H1KqJN/iPRDIZi2da94DOI6DmOGx71+toYsLpevBPYK2/iqr+V1VvV9XbsXxlnnA7uKTlo99C8QarXvfc/z398xt+epofVb7ETHLyD4OmWH/XJ8cwzekSfCdVXdX4Tvu+3q5ElOwc/gzetZdFm/JYaD/p6xP8lgVQXe5aaIYISYSFPhKF+nNl45tWCXKCc7oEf6qlWlo4GYgBUIU37oTaChh2DfSbENp+uV2g65nWflsXuhqiIQLql+ozk5y8p/Nw630o2w97PvE6Gtc5XYJfJiI3Nb5TRL6N5fNucJL1s2HTXMtM7JKHwtt3UPJVCMQNDWPwpkTSc4LNx5JgmOZ0Cf5/gW+IyEIRecy+vQt8C7jN/fCSiGAzsYtmQE6n8PZvGIc35mO+w4zB+4skumZ1ygSvqvtV9RzgPuAz+3afqp6tqvvcDy+JWPiwVSvd9YwTTpHh0HGI1UMsL4ZdS52PzxAZgQCU7rK2zRCNP+h9HmTmwcH1ULLF62hcJVQvmgWq+qR9e8ftoJKOfatg8f9Zq83Um4mFi0jSVQjEBWX7IFAD2e0hw9mVmQwRkpoOAy62thO8F29mo3pNIGDVvGsdnHWT1YOPlCT66Rk3GJMxfzIoOWaAmwTvNcv/Crs+tmY5Trg7Oq2eZ0OLNlCy2ZiP+QVjMuZP+k+0zMd2Lk5o8zGT4L2k7CDMu8fantyEmVi4pKbBANt8zAzT+IP6EknTg/cXWXlB5mNzvY7GNUyC95J50y0zsX4TYOgXnNFM4vUnfUnDQh8mwfuOJLhmZRK8V2x7H1a+YJmJXfZo82Zi4dJgPvYxlB04/fMN7mLG4P1LfWdoyztQU+FtLC5hErwX1FZbqzQBjPvRqc3EwiUzB/pcgDEf8wlmDN6/5HWHLiOhphy2Jqb5mEnwXrDoN1C8Edr1h/NCMBMLlySpEPA9qmYtVr+Tbw/TJOgMcJPgY82hbfDeL63tKY9BWqbzbQy0f3puNeZjnnL8INRWWpVNWbleR2NoikHBM8ATz3zMJPhYogpz7rRO+uHXWis1uUFuF+hWYLWzdYE7bRhOT/34u+m9+5dOw6wL4McPwO7Es9cyCT6WrJtlrQeZmQeTHnS3rQZDJTNM4xnmAqv/CTYfS8BhGtcSvIj0EJEFIrJWRNaISFKbk6XUlp8wE5sYgZlYuNSPLW405mOeYRJ8fJDA16zc7MHXAj9S1SHAWOB7IjLExfZ8TdcNz1lrc3Y9Ewq+4X6DHQdDm96W+djOj91vz/B5zEIf8UGvc62JT8UbEs58LM0tYVXdC+y1t4+JyDqgG7DWrTYdY8cSOm/6N5QvdEavtoqO2/4bnZlYuIhYvfjFT1k/PXud7X6bhpPx2Ri8qvLKit20KDe/6E4iNR0GTIJVL1qTns79gdcROYZrCT4YEekNnAEsiUV7UXFkB/ztKrrVlMN652QFYMx3oeso50RPx6DL7AQ/BybNjF27BgufLbY9Z/U+fvivlXTLSWXC2QEy0swluAbyL7MS/IY5CZXgRVXdbUCkFfAu8KCq/reJx28Gbgbo0qVLwaxZsyJqp7y8nOzs6O1Y+308jdb7F1GaN4TyDiOj1qunUtM5PPAraFoIa6yGyGmPOVDHyLe+SFrNUVaPf46qnFMnGqdeQ7f03NB0TU+VUXOmkFpXyYpLXqUuI8fz+H61+Agf7KwE4KvDWvHFwa0c0fX7exKKZkrNcUbOvRrRACsn/Ye6zFP7QvnpmAsLC4tUtbDJB1XVtRuQDswFbg/l+QUFBRopy5Yti3jfBtbNVr0nV/XBbrrygzej1wvCkfgi0fzPzdYxvf+4M3ph4Nkx+0GvrNh63R/qrhoIRK8XJdW1dTrsnje1112ztddds3Xg3W/o9uLjjmj7/T0JWfOvV1nv2SfPO6MXBtHoAcu0mZzqZhWNAM8A61T1cbfacYyqMmvBa4AJ06jJau9tPE6RwBUCvqY0aPzdKZ+hKFiy9RDHKmsZ0LEV43pmUVUbYMZrq+s7YgZIyPUU3ByEOxe4HpggIivs22Uuthcd7/4cju6yvCnO+rbX0ThHv4ssQ7NdS435WCzxWYnkvLXWCpsXD+nEDSNzyMlKY+GGg7y52qy82UB9gk8g8zHXEryqfqCqoqojVHWUffPnV+P+NfDRU4BYVS6pMbn2HBsyW0Ff23xswxyvo0kefGQypqrMX2d9uV88pBNtslK5c/IgAO6btZayqlovw/MPed2gyyjbfGyh19E4grmMHgjA7B/aS+Z9y5rin2g0/PQ0CT5m+KgHv3bvUXYfqaBDTiYju7cG4KujezKyex77jlbyq3kbPY7QR9R7xCfIMI1J8Cueh51LoGVHmDDd62jcIT/YfOy4t7EkCz5ykZy3dj8AEwd3JCXFuh6QmiI8ePVwUgT+/OE2Vu8u9TJE/9BgW5AY5mPJneCPF8O8Gdb25IehRWtv43GLnM4nzMe2GPOxmOCjHnx9gr94yMn2GMO65XHDOb0JKEx7ZTWBgLngerL52DKvo4ma5E7w82ZAxWFrgYxhX/Q6GndJwAoBX9MwBt/L0zB2H6lgzZ6jZGekck6/z1eG3X7xQDrlZrJi5xFeWLrDgwh9hkhQ5Vn8m48lb4L/7ENY8Xdrebspj/uilM1VBhnzsZhRcQSqSiE9G7LbehrKfLv3fv6ADmSlf94iIycrnRlThwLw8znrOXisKqbx+ZIE6gwlZ4IPXjLvvNuhfX9v44kFHQZBmz5QXmJdczC4R7DJmMcdh/nrmh6eCeay4Z25YGAHjlbW8tAb62IVmn/pdY5tPrYRijd7HU1UJGeC/+i3cHA9tO0L5/3Q62hig0jCVQj4Fp+YjB2trGHx1hJSBCYM6tjs80SE+68cSmZaCi8v382iLcUxjNKHpKbDgEus7Tg/V5IvwR/eDu/+wtqe8hikO+cN43vyg2a1mhmM7uETk7GFGw5SU6cU9m5Lm5YZp3xur3Yt+f6F1i/Zaa+spqo2yYfxGqppTIKPH1ThjR9DbYV1UbXfBK8jii09xljrgx7aYv38NLhDQwWNtz34+uqZSacYngnm5gv60rdDS5vy83cAACAASURBVLYePM4f3t3qZmj+p/9ESEm3hjOPx+8vmuRK8Otnw6a5kJkLlzzkdTSxJzUNBk62thOgQsC3HNlu/fWwB19dG2Dh+hOzV0MhMy2VmVcOA+C3CzazvSSJ50xk5UKf80EDVmFCnJI8Cb6q7MSSeROmW7XhyUgCVQj4loZJTt4l+CXbSjhWVcvATq3o1a5lyPud0789V5/RzTIje3VNcpuRJYBRX/Ik+IUPw9HdltfEWd/yOhrv6DfBNh9bBsf2ex1NYuKDSU7zm5ncFAo/u2wwuVlpvLvxIG+sSmIzsmDzsepyb2OJkORI8PtWw+L/s5bMu/yJ2CyZ51cyW0Hf8YDG9U9Pv5JSW25NnkvNhJYdPIlBVYNmr4b/S7VDTmaDGdn9s9dwrLLG0fjihtyu0PUM65rdtne9jiYiEj/Bn2Qm9m3rDUt2EqRCwI9klNu/ilr3gBRvTq81e46yp7SSjjmZjOh26pWJmuOro3syqkdr9h+t4vFkNiPLj+9ZrYmf4Jf/FXZ9DK06wYRpXkfjDxrMxxYa8zGHyaiwE7yHNfD1vfeLBndqMBcLl5QU4cGrh5Ei8JdFnyWvGVl9go/TGeCJneDLDsK8e6ztyQ9bs9MMtvlYoW0+9o7X0SQUmeX2mLWH4+/hlkc2x9Cuedx4Th8CCne/vIq6ZDQj6zTUei+PH7SuW8UZiZ3g582AyiPQ90IY+gWvo/EXCVAh4EcaevAe1cDvOlzO2r2WudjZ/dpFrXf7pIF0zs1i5a5S/vFxEpqRiUB+/Qzw+BumSdwEv+19WPkP62LXlMc89wTxHflB5mN1ZkUfp8ho6MF74yL5tr1y0wUDmzYXC5dWmWnMuHwIAL94cz0HjlVGrRl3xHFnKDETfG01vP4ja3vcj6BdP2/j8SMd8i0vnopD1jUKgyNkNvTgvRmiac77PRouHdaZ8fkdOFZZy0OvJ6EZWc+zIas1lGyC4k1eRxMWiZngP3oSijdA235w3v96HY0/EYn7CgE/0lBF48FF1tIKy1wsNUW4ML95c7FwERHuv2IYmWkpvLJiDx9ujt+p+xGRmg4DJlnbcVZ5lngJ/tC2k83E0jK9jcfPBM9qTeYZi05RU0F69WHLw8SDmdILNxygNqAU9mpzWnOxcOnZLptbJ1hmZNOT0YwsTodpEivBq8KcO63qkOHXQr8LvY7I3/QYAy3awqGtcHCD19HEP/UuknndPJlM58bwTDA3nd+Xfh1asrX4OL9PNjOy/hOtxYF2LrGq8+KExErw62bBprcgMw8mPeh1NP4n2HwsDisEfEepdxYF1bUB3t1gJZ5JEcxeDYXMtFRmXjUcsMzIPitOojkUmTmW+ViczQBPmASfUlt+wkzsoumQ404vJuGI05+evqRhoY/YJ/h6c7H8Tjn0bJftWjtn92vHF87oRnVtgOmvrk4uM7I4NOpzLcGLyLMickBEVrvVRjBdNzwHx/ZA1zOh8JuxaDIxqDcf272MtMpDXkcT33hoMub28EwwP5timZG9v6mY11ftdb0931A/A3zLAqQ2PspF3ezBPwdMdlH/BHs/peO2/1pmYlN/ldxmYuGS0dI2H4PW+xd5Gkrc07CSU2wraFS1wT1yYgwSfPtWmfzk0sEA3D9rLeU1Adfb9AVB5mO5xUVeRxMSaW4Jq+p7ItLbLf0GbDMx0QCM+S50HeV6kwnHoMtg01xyNv0X5jv3k7vrvn1weJZjem5oOqpXv5h5jHvwTpiLhctXzurBi0U7Wb7jCC+sLmPc2Jg0GzZlVbW8sGQHfVIcqvrJnwJ7ltN63yLgB45I/n3Jdj7dWMawkXVkpjnbOXUtwYeKiNwM3AzQpUsXiorC+2Zsu2s+fXYvoyqjLWvbTCEQ5v7NUV5eHnYssdRzUnPL7g58QVNoW/EZfPCr6AOz6QLg8KL0Tms6racIq3Yfp6Ykdp/Df645BsCoDiksX/6JI5qh8LX8VFbuhDmby/n3/MX0a5MetSY4e678dmkpCz6rYGCbVPKylpES5Yz2rEAfhgJt9izg0w/mUNMiuvkGhyrqmPlmMRW1Sr83l1DQxdmybs8TvKr+AfgDQGFhoRYUFIQnMHI4tMtg1xE4Y+z5jsVVVFRE2LHEUM8pzeraAHe9+z6v1dzFCNlKtzYtuG50TyI0ITyJ3bv30K1b1+iFXNR0Wm/L0QxGnHuJY3qhvMfTPngfgOsuGEZBCBOcnPosFgDrKtbypw+28fz6Wl6+ZQypDnxwnIpvydYSFny2GICNh+vYpp34UmG0w2cFcOAVUtfNYsTuv8OXn49K7fv/+ISKWqWwSyY3TTkbcdhSxfMEHzVpGXDeDznicO84WfjTB1vZfKCMmnZjWFU2ktLiAOnZIxw4EWBfURHdHP5Sc1rTab3SGH8Odx0uZ93eo7TMSOUcB8zFwuV/Lx7If4u28+muUv6xZDvXn9075jE0RXVtgGmvWPUdhb3asGz7YR6es46JQzrRNtpJYJMfoW7jPFLXzYKNc2FgZF/o7208yOxP99IiPZVvnZHreHKHBCqTNITPzkPl/OZty1tj5lXDuHFkDgAPv7GOQ8ervQzNECL1F1fPH9jB8fHbUGiVmca3RuUC8Is3N/jGjOxPH2xl04EyerfL5vlvj2F4xwwOl9fwyBwHvHTyurMn/xvW9ht3RLScX2VNHdNftb6AfnDRADq2dOe9c7NM8gXgIyBfRHaJSBIvhOo/VJV7XltDZU2AK0Z2ZdyADozrmcU5/do5dyIYXGfeutiVRzbHmG6ZTBjUkWNVtcyc7f3nJrjj8sBVw8hKT+WmM3PJSE3h38t28fG26MuBD/T5AnQabpXGvv9o2Ps/vXAL20vKGdipFd8e1yfqeJrDtQSvqtepahdVTVfV7qr6jFttGcJn7pr9vLP+ADmZaUybapW8iQgPXDWs4URY+pmpi/czpRU1LNl6iNQUYcIg58zFwkVEuO+KoWSlp/Dayj28v8m7qfzBHZfL7Y4LQLecNL57QV8Apr2yipq6KEs7U1Jh6uPW9oe/gQPrQ95168EyfrdwCwAzrxpOeqp7AylmiCYJKauq5b5ZawD48eR8OuZkNTzWr0OrhhPh7pcdOBEMrlFvLnZW7za0znbWXCxcerTN5tYJAwCY8eoaKmu8MSML7rhMnzL4pMduubA/vdpls3F/Gc98sC36xnqMhoIbIVBj2ZOHMKtXVZn+6mqq6wJcW9Cd0X3aRh/HKTAJPgl5Yt5G9pZWMqJ7Hv9vzOcXpnD8RDC4wlsNs1dj71zZFDeN60v/jq3YVnyc3727JebtH2/cccnNOunxrPRU7r9yGABPzN/IzkPhj51/jovugez2sP0DWPnP0z79tZV7+HBzCa2z0/npZYNP+/xoMQk+yVi75yh/XvQZKQIPXjW8ybK24BPh1/M3seuwAyeCwVGCzcUuHuwP36WMtBQevMr63Dy9YAvbYmxG9sT8U3dcwFrpauqILlTWBBq+DKIiuy1Mmmltv3U3lDc/rFlaUcMDs9cC8LNLB0dfzRMCJsEnEYGAcvcr1uLJXz+7N8O7Nz/r8YKBHZgyogsVNXXc+9raGEZpCIXFW0soi4G5WLiM6duOL57Zneq6ADNiaEa2ds9Rnv3w1B2XeqZPHUKrzDTmrzvAW2v2Rd/4yK9Ar/OgvATm39vs0x6du4HismoKe7XhmoLu0bcbAibBJxH/XGpNLe+Yk8ntkwae9vkzGk6E/c6cCAbHiKW5WLj87LJB5LVI5/1Nxcz61H0zskBAmRZixwWgU24Wd9if/3tfW8PxqijXJBaxLrimpMMnf4EdSz73lBU7j/D8ku2kpQgzrx5GihMzCUPAJPgkobisqqH0ccblQ8jNOv208k65WfzIyRPB4AiqynwflEc2R7tWmfz00kEAPDB7LUcra1xt71/LdvLJjiN0CLHjAnD92b0Z3i2PPaWV/PptB9ZZ7ZAP59reNLN/CHUnjrm2LsDdL69CFb51Xh8Gdc6Nvr0QMQk+SXjojXUcraxl3ID2TBneJeT9rh/bi2HdctlTWtlQW2zwltW7j7K3tJJOuZkMj5G5WLh8qbAHZ/ZszcFjVTw2173VwqyOi1WiOGNqaB0XgNQU4cGrhyECz3ywjfX7jkYfzLg7oHUvOLAGlvyu4e6/Ld7Omj1H6da6BbdNHBB9O2FgEnwSsGhLMf/9ZDcZaSk8cOWwsKZEp6Wm8OBVw509EQxRMW+tNVw2cXCnmP3UD5eUFOHBq62x8L8u3s6nu4640s5Db6yjtKKGcQPaM3VE6B0XgBHdW3P92F7UBZS7X15NIBDl9YKMbLjMnvS04GEo3cX+o5U89tZGAO69YijZGbF1hzEJPsGprg0w3fbk+P6F/endvmXYGiN7tOZrY3pRG1CmOXEiGKJi3roDgD+HZ4IZ3CWXb53XB1W4++XV1Dn8ufloS0nEHZd67rgknw45mRRtP8y/l+2MPqiBk2DwFVBzHObcxf2z11JWVcvEwZ08eb9Mgk9w/vj+VrYcPE7f9i35jj2BKRLuuCSf9q0yWbb9MC8WOXAiGCJi56ET5mJne2AuFi63XTSArnlZrNpdyvOLtzuma5mJrQLge+Mj67gA5GalM82eEPXIm+spKauKPrjJj0BGK1g/m4rVr9MiPZV7rxgSvW4EmASfwOwoOdmTIxozqrwW6Uy3LQ0enrPemJF5RP3F1QvyvTEXC5eWmWncc8VQAH45dwP7jzpjRhbccfnu+Mg7LgBXjOzKef3bc6S8hofnhG450Cx53ag5/ycA3J/+HHdc2J3ubbwpZTUJPkFRVWa8tpqq2gBXjerKuf3bR615xciunNu/nXUivOG9qVQy4ufyyOaYNKQTEwd3pKyqtmGiTzQ42XGBIA+mtBReKtrFkq0lUcf42+MTWBvoRXcp5sbaF6PWixST4BOUN1fvY+GGg+RkpXH3FGd+HooID1xpmZG9WOSMK58hdErLa1iyzTIXuzCEhT38gohwz+WWGdnsT/fy3sbIzciCOy5XOtRxAejTviX/c0E/AKa9sprq2sg9mDYfKOPp97bzs5pvoQipi38LB7zpEJkEn4CUVdVyrz0N+87Jg+iQ49wyYH07tOJ/xtefCKuiOhEM4bFgwwHqAsro3m09NxcLlx5ts7ntIqtGffqrqyM2Izu54+Ksl8v/jO9H73bZbDpQxp8+2BqRhqoy/ZXV1NQp+QUXIgU3QqAWZt8ekhmZ05gEn4A8/tZG9h+tYmSP1nx1tPMLQNefCMaMLLb4wfs9Gr49rg8DO7Vie0k5Ty8M34zMckG1hnjuvORkF1QnyEpP5QHbS+c3b2+KyIzs1RV7+GhrCW2y0/nJpYNgom1GtmMRrPiHo/GGgknwCcbq3aU8t2ib7ckxzJE1MhtzkhnZ2w658hlOSVVt3QlzsThN8OmpKcy8ajgAv1u4ha0Hy8La/1fzNrLvaCUju+fx1WbMxKJl3IAOXD6yK5U1Ae55bU1YXjql5TXMfN36AvrpZYNp0zIDWrSBSx6ynvDWtFOakbmBSfAJRF1AufuV1QQUbjinN8NcnOV4/sATJ8K9YZ4IhvBZvPUQZVW1DOqcQ4+2/jEXC5fRfdpybYFlRjY9DDOyNXtK+fOHdsfl6lObiUXL9CmDyclM4531B5i7Zn/I+/1i7nqKy6oZ3bst15wZZCY24kvQexxUHIL597gQcfOYBJ9AvPDxDlbuPEKn3Exuvzg0T45oqD8R3l5/oMGb3OAO9bNX47X3HsxPLxtM6+x0Ptxcwmsr95z2+YH6maYKXz/b3Y4LQMfcLH48OR+A+2atoSwED6blOw7zj493NG0mJgJT6s3I/go7FrsV+ucwCT5BOHisil+8adXw3nP5UHJC9OSIho65WdxxiX0iGDMy11BV5q+Nj9mrodC2ZQY/u9S6QPrA7LWUVpzajOyFpTtYsdNyQf1RiGZi0fL/xvRiRPc89pZW8sS8jad8rmUmthpV+Pa4vgzslPP5J3UYCOf9r7XdyIzMTUyCTxDqzcQuGNiBS4fFboWfr43t5awrn+FzrNpdyr6jlXTOzfKtuVi4XFPQncJebSguq+bRU5iRHTxWxc/rzcQuHxKTjgvYZmRXDSdF4M+LPmPtnuY9mP7y0XbW7rXMxH5wUf/mRcf9CNr0hgNrYfH/OR90E5gEnwAs2lzMy8t3k5mWwv1XDo3IkyNSGrvyrdtrzMicZr49/DVxSMeYvrdukmIPZaSlCM8v2c6KnU2bkT1sd1zOH9ghLBdUJxjePY+vn93bvra1qkkPpn2llTz+lvUFdd/pzMTSW8Blj1nbCx+GI+5bfpgEH+dU1dYx7VXLTOzWCf3p1S4yT45oGNG9NV+3XfmmvWLMyJym/vrGRJ8szecUgzrn8q1x9WZkq6httMD7oi3F/Hd5vZlYbDsu9dw+aSAdczJZvuMI/1z6+YR8/+w1HK+us2brhjJ8NmAiDLkKasphzl0uRHwyJsHHOX94dytbDx6nb4eW3HR+dJ4c0fAjp135DADsP17L+n3HaJWZFhfmYuFy20UD6Na6BWv2HOVvQWZkVbV1TAtyQfWi4wKWGdn0qdZM8J+/uZ7iIDOyBRsO8MaqfbRIT23w2wmJyQ9bZmQbXof1bzgd8kmYBB/H7Cur5ckFmwGY6YAnRzQEnwgPz3HIlc/A0j3W63jBwPgwFwuX7Iw07rWT42NvbWRfqWVG9sf37I5LlC6oTjB1RBfGDWhPaUUND9keTFV1ygz7l/MPL7a+pEImtytMmGZtz7kTqt1bnNzVBC8ik0Vkg4hsFpGfuNlWsqGq/Gn5UaprA3zhjG6c088ZT45ouDzoRHDElc/A0t1Wgk+E6pnmuHiI5ZVeb0a2r6yWJ9/xR8cFgjyY0lL47ye7+WhLCf9ZV8bOQxUM6pzDN87tE77oWTdB5xFQuhPe/bnzQdu4luBFJBV4CrgUGAJcJyLemCInIG+s2sfyfdXkZqXxM4c9OSJFRLj/yhOufGsOGkvhaDhSXs3a4uq4MxeLhHuvGEqL9FReX7WXBz843OCCeo5DZmLR0rt9S7433qqQufM/K3l1vdXrnnnVMNJTI0ijqWkw9QlA4KOnyDrqjuWHm+tHjQY2q+pWABH5J3AlEL1faBC7Dpfz+3e3cvDgUV7ZudoxXb/rzVltTXy569JBtG/lnJlYtPRp35Jbxvfjifmb+O3SUjZVOnfM4P/3xUm9vaUVBBTO6duWvOzYlAd6RbfWLfjfiQN4eM569hyrc9QF1Sm+O74vr67YzdZiK7l/5aweFPZuG7lg9wIo/CYse4aeq56AC74AKc72ucWtKeYicg0wWVW/bf9/PTBGVb/f6Hk3AzcDdOnSpWDWrFlhtbP5UA13vR29f3M80q91Ko9MbE+KQ9UF5eXlZGdHPw2+uk65/a1i9pZF5hhoOJlvn5HDpf2du8jo1PvstF5tQPnx/BJ2lNZy05m5TO7nvxg/3V/Ffe8dJidDeHJyB3Iyo0vIqTVlDH3nBtKrD7P5rAco7Xxu2BqFhYVFqlrY1GOxXQG2CVT1D8AfAAoLC7WgoCCs/Xseq+L+VnvZsWMHPXs655zod70UETrX7eeswibf14goKioi3Ne/OV4cUM5f5hXRo0cPR/Tq8fv74rTewb27+MGV50Q2DNAMTr7PTuu9mF/Jy+8W8Z2p5zhaFulUjAXAoPwSDu7czPhzzoo+MIDcX/PZxtX0v+xWy9bAQdxM8LuB4LO7u32fo3TIyeTrZ/emKKOEgoLejun6XQ+gqKjYUT0n6d4mm8n9sp0/Zp+/L47rFZU4mtz9TqfcLM7qmuXrCV1n92tH0ZHPnBMcciUlFd3p7cIxu/nJWQoMEJE+IpIBfAV4zcX2DAaDwRCEaz14Va0Vke8Dc4FU4FlVXeNWewaDwWA4GVfH4FX1DcDdqVoGg8FgaJLkGdwzGAyGJMMkeIPBYEhQTII3GAyGBMUkeIPBYEhQXJvJGgkichDYftonNk17wMnCcL/ruaHpdz03NJNNzw1Nv+u5oeknvV6q2qGpB3yV4KNBRJY1N103EfXc0PS7nhuayabnhqbf9dzQ9LtePWaIxmAwGBIUk+ANBoMhQUmkBP+HJNNzQ9Pvem5oJpueG5p+13ND0+96QAKNwRsMBoPhZBKpB28wGAyGIEyCNxgMhgTFJHiDwWBIUEyCNxgMhgTF8yX7DAaDIdkQkdGAqupSERkCTAbW2xbrzrUTj1U0IvI48B9V/dAhvR8AL6vqTif0TtPWN1T1zw7onAeMBlar6lvRR3aS9sWqOi9KjT7AGcBaVV0fhY6jJ4KTem59bkSklR1XD6AO2Ai8paqBCPX6Al9opPcPVT0agVZ7VS0O+v9r2J9D4I8aQUJxMj5bT4BrAQVeAiYAVwLrgd+F+zq6kG/uAS7F6mDPA8YAC4CLgbmq+qAT7UD8Jvh6z5oOwL+AF1R1eRR6pcBxYAvwAvCiqh50ItYm2tqhqmGvyiwiH6vqaHv7JuB7wMvAJGCWqj7iZYwi8oqqXmVvXwk8ASwEzgEeVtXnIojD0RPBBT3HPzci8iXgDuBT4EJgEdZQ6nDg/6nqqjD1fgBMBd4DLgOWA0eAq4FbVHVhmHqfqOqZ9vY0YBzwD7uNXar6Qy/jszWfBjoCGcBRIBNrudApwH5VvS1MPafzzSpglB3XPqC7qh4VkRbAElUdEan251DVuLsBy+2/A4HpwBqsb+d7gIGR6GGdRJOAZ4CDwJvADUBOBHqfNnNbBVRFc8z29lKgg73dElgVgd5rzdxmAcejjG8R0Mfebg+sjPCYV2Et95iNdaLm2ve3AD71gZ6jn5ugz0520Gs3194eASyK9Jjt7Wxgob3dM/g9i/B9/gRoaW+nR/g5dDS+es2gmEqADPv/tEjfZ/uvY/mmqW37/xWRHHNzt3gdg1cAVd0IPAA8ICIjgOuwlgjsH66eWj/b3gLeEpF0rJ7edcCjWN/c4dAJuAQ43Oh+wUp+kZAiIm2wEoqo3VNU1eMiUhuB3jjga0BZEzGOjkAv+Kdgmqpus+MrFpGIhhaAWlWtA8pFZIvaP9lVtSJCTaf1nP7cgPX6V9jbx7F6oqjqpyKSG4EeWImtDqvH2MrW22HHGy4tROQMrM9hqqoet/VqRKTOB/EB1AbFtFRVq+3/ayN9n+39nco31SKSrarlQEH9nSKSB0R6rjRJvCZ4aXyHqtb3kn8arZ6q1mD3aEUkOwK92UArVV3xuYZEFkagB5AHFGHFqiLSRVX32uO1n3s9QmAxUK6q7zYR44YI9EaKyFE7lsyg+DKwes2R4PSJ4LSe058bsBLGmyLyHtY4/It2jG0btxcifwKWisgSrC/1n9t6HYBDEejtBR63tw8Fvc/tsBOrx/EB7BORVqpapqqT6+8Ukc5AdQR6Tueb81W1ytYJ/tylY/36c4x4HYNvpaqNe57R6A20v53jDjuRdKrvMfsNEWkNDFbVjyLYN7P+RGh0f3ugi4Y/Hu20niufGxG5DBiCNbQ1z74vBUhvKv4Q9IYCg7EuyEd8wfs0baQCmfaXZ7j7uh6f3U5LrCGlA2Hu52i+iWVbcZng6xGRQoKuvLvx4Yjlm+s1InKFqr7mVz1bs62qRtSzE5HWqnrEwVgc1QvSTVPVWnu7FTAI2BrFcdcPu3Wz79oNfKxRnPwikm7/Ygm+76QKmyi0b1HVp6PVCdJrhTV+vtWJ98tpvUbaERVhNEdcDtGIyAXAY1hX2wuAD4E2IlIDXK/Olq2txbrgE058I7Dc4boBc4C7VPWw/VhDNYyXmiLyhcZ3AU+JSBqAqv7XSz1bc5qqzrS3hwCvAOl2wvqyqi4JU7LYHiJ7AavsLdqT02k9RORG4DERKQFuA54CtgEDReROVX0hTL1JwNPAJqzEDtAd6G8n0rBKbEXkQuBvQJaIfALcrKqf2Q+/BZwZpt7tje8CfioiWQCq+vjn9zqt5tOqeou9fR5Wlc8WrGP+joZZEuuCXuNjbngI+xqEU8RlgscqwZukqgfFqrd+XFXPFZGLsaoZJoUj5sIL/jRwL9Y497eBD+ze7BascbZIcFrzX8Bc4AAnxhhbApdjXVQKNyE7rQdWbfRMe/uXwG2qOkesWvYnsEoww2Gdvd91wC9E5AOs5Pyqqlaccs/Y6AH8CMgHcoCVwBmqukVEOmGVdoaV4IFfAxODkjDQME/hDayhkXD4BXCJqq4RkWuAeSJyvaouJrJrBPfZcawJ2j8V6/gjZWzQ9gPAVar6iVj19v+22/NS7yGsz3NT1yycdRdwsiQnVjeCSp2wPgyfBP2/JgK9Sqw37p4mbkci0FvZ6P8LsXpQY4Nj9VITOAt4G/ifoPu2RfGeOKpn7x/8vjYuJ4ukxC9YrwXwJawvnhKsiTWe6tk6K4K29zR6LJISv01YVU2N788ANkeg1/hzOBTYAFwV4eewJ9aF5J9zojx0q4Ofm6LmHvNQbxFQ0MxjO6M59sa3eO3BLxORZ4B3gCuwJtTUX3CMpGLjE+AVVS1q/ICIfDuSAEUkT1VLAVR1gYh8EfgP0DYSPac11ZrJeTFwq4gsAO7i5FJHT/Vs+orIa1g9u+5BFTAQ2a+Whh6mWj3sfwP/tqtorvKBHsAOEXkYqwe7XkQew/rSmIhVwRIuz2JVqfwTqB+67AF8BevXbrjUiEhnVd0HoFZP/iKsyrF+4Yqp6g7gWrEmx80TkV9FEFNjBonIp1jvT28RaaOqh+0L1Rk+0PsGzVcIObsWrf2tEVeIVR97E3alAfCsqtaJNROso6puD1MvHzikTcxCFJFOqro/TL2vYvVCFje6vycwXVVvCkfPLc0gjW7Ar4BCVe0bqY7Teva1lmCKVLXMHq64RlWfClPvDlV9NNJ43NazNXOxZikr8FusUskbgR3AA6oadpK3r19cwckXWV9T1bURaE0EDqrqykb3twa+p1FMs7cvXt4DjFHV86PQ6dXorr2qWm1XEJbqNQAABR9JREFUS52v4V9fclQvlsRlgjcYDIZ4xf6F91OsX3kdsb7MDwCvAo+og5U5cWkXLCKfiMg0+yKHE3qFIrJARJ4XkR4iMk9ESkVkqViz9pzUG+VCjGFrNqN3xOFjjljvFJpOH7Nb77OTxxzN+9JKRO4XkTV2bAdFZLFY1TqRxOdrvTiJ8d9Ys9zHq2pbVW2HdU3tsP2Yczg5oB+rG1bZ2KNYP1s/Bn4IdI1C72NOTDHfifXzH+Ai4COv9eIhRnPMvj3mV7GGeLoDt2N5qQwA/gI8lGh68RAjsCGSxyJ6LZwUi9WNk69qj8MqIdyH5Qx4cwR6weY/O5p7zCu9eIjRHLNvj7lx1ctS+28Klk1yQunFQ4xY8wXuxJqBXn9fJ6zChPmRHHNzt7gcoglGVd9XaxJCN6xSq7MjkKkUkUkici2Wz0u97e0FWLNkvdaLhxjNMfvzmI+LNTkHEbkCu3pDLQ+USOrW/a4XDzF+GWgHvCsih0XkEFYlYFusUlvncPLbIlY34J8O643EmqQzB2ta+K+xZsmuAc7xWi8eYjTH7NtjHoE17HMY+ADb3hbL6fIHiaYXRzEOwip9bdXo/smR6DXbjpNifrgB30gmvXiI0RyzPzWTTc8vMQI/wJoc9grwGXBl0GMRTYRs7pZwZZLisFmP3/Xc0PS7nhuaftdzQzPZ9NzQjERPrBWdzlZrTkdvrGUF/6aqvxaR5aoaUQVWU8TlTFaxZpU1+RDWxYqE0nND0+96bmj6Xc8NzWTTc0PThRhT1HaoVdXPRGQ88JJYE6oive7QJHGZ4HF+xSS/67mh6Xc9NzT9rueGZrLpuaHptN5+ERml9oJAdk9+KpatxPAI9JolXhO80ysm+V3PDU2/67mh6Xc9NzSTTc8NTaf1vk4jJ0m1/P+/LiK/j0CvWRJuDN5gMBgMFnFfB28wGAyGpjEJ3mAwGBIUk+ANCYmI3G2bQ30qIitEZIyLbS0Ua31gg8FXxOtFVoOhWUTkbGAqcKaqVonl2x3JwgwGQ1xjevCGRKQLUKyqVQCqWqyqe0Rkhli2u6tF5A8iItDQA/+ViCwTkXUicpaI/FdENolI/aLfvUVkvYj83X7OS2KtIHYSto/MR2JZWr8o1iIWiMgjIrLW/kXh6CIhBkNzmARvSETeAnqIyEYReVpOrAz1W1U9S1WHYa2hOjVon2pVLQR+h2UP+z1gGHCjiLSzn5MPPK2qg4GjwC3Bjdq/FKZhLXJ9JrAMuN3e/2pgqKqO4MRC4gaDq5gEb0g47FmCBcDNwEHgX2ItznChiCyxp4pPwFowup7X7L+rsBZu32v/AtiKtYYpWAsif2hvPw+c16jpsVjLSH4oIiuAG4BeQCnWwu7PiMgXgHIMhhhgxuANCYmq1mFZsC60E/p3sFwBC1V1p4jcC2QF7VJl/w0Ebdf/X3+eNJ400vh/Aeap6nWN4xGR0ViLdlwDfB/rC8ZgcBXTgzckHCKSLyIDgu4aheXeB1Bsj4tfE4F0T/sCLsBXsaxjg1kMnCsi/e04WorIQLu9PFV9A2v1sZERtG0whI3pwRsSkVbAkyLSGmtK+Gas4ZojwGqs1b+WRqC7AfieiDwLrAX+L/hBVT1oDwW9ICKZ9t3TgGPAqyKShdXLvz2Ctg2GsDFWBQZDCNi2rrPtC7QGQ1xghmgMBoMhQTE9eIPBYEhQTA/eYDAYEhST4A0GgyFBMQneYDAYEhST4A0GgyFBMQneYDAYEhST4A0GgyFB+f+0SfByWslZywAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Movie reviews"
      ],
      "metadata": {
        "id": "BDLigfEpAa6R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.corpus.movie_reviews"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3znkLd0NAfGI",
        "outputId": "8450aa44-80c7-48a2-ade3-e3fdf1e6ea4f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<CategorizedPlaintextCorpusReader in '.../corpora/movie_reviews' (not loaded yet)>"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('movie_reviews')\n",
        "from nltk.corpus import movie_reviews"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UaKDnAk0Alsu",
        "outputId": "936d5957-4ee2-4216-d625-7ca24be96686"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package movie_reviews to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/movie_reviews.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "file_ids = movie_reviews.fileids()"
      ],
      "metadata": {
        "id": "ELsUcvcnAoHp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#file_ids is a list of file names\n",
        "\"\"\"\n",
        "['neg/cv000_29416.txt',\n",
        " 'neg/cv001_19502.txt',\n",
        " 'neg/cv002_17424.txt',\n",
        " 'neg/cv003_12683.txt',\n",
        " 'neg/cv004_12641.txt',\n",
        " ...]\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "YOYw-EBNlsSN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(file_ids)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ErLBD5j4BUhG",
        "outputId": "d5b5d36d-9004-452d-e97e-e4295381ea9c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2000"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "neg_ids = [id for id in file_ids if id[:3] == 'neg']\n",
        "pos_ids = [id for id in file_ids if id[:3] == 'pos']\n",
        "print(len(neg_ids), len(pos_ids))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I0vkQmifCFjC",
        "outputId": "7483db92-e2e0-4b1f-82e7-3bb56c64ddb1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1000 1000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "movie_reviews.raw(file_ids[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 249
        },
        "id": "m2gnftT2BCMx",
        "outputId": "53076b18-dc04-428d-a317-df79e1f4c66d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'plot : two teen couples go to a church party , drink and then drive . \\nthey get into an accident . \\none of the guys dies , but his girlfriend continues to see him in her life , and has nightmares . \\nwhat\\'s the deal ? \\nwatch the movie and \" sorta \" find out . . . \\ncritique : a mind-fuck movie for the teen generation that touches on a very cool idea , but presents it in a very bad package . \\nwhich is what makes this review an even harder one to write , since i generally applaud films which attempt to break the mold , mess with your head and such ( lost highway & memento ) , but there are good and bad ways of making all types of films , and these folks just didn\\'t snag this one correctly . \\nthey seem to have taken this pretty neat concept , but executed it terribly . \\nso what are the problems with the movie ? \\nwell , its main problem is that it\\'s simply too jumbled . \\nit starts off \" normal \" but then downshifts into this \" fantasy \" world in which you , as an audience member , have no idea what\\'s going on . \\nthere are dreams , there are characters coming back from the dead , there are others who look like the dead , there are strange apparitions , there are disappearances , there are a looooot of chase scenes , there are tons of weird things that happen , and most of it is simply not explained . \\nnow i personally don\\'t mind trying to unravel a film every now and then , but when all it does is give me the same clue over and over again , i get kind of fed up after a while , which is this film\\'s biggest problem . \\nit\\'s obviously got this big secret to hide , but it seems to want to hide it completely until its final five minutes . \\nand do they make things entertaining , thrilling or even engaging , in the meantime ? \\nnot really . \\nthe sad part is that the arrow and i both dig on flicks like this , so we actually figured most of it out by the half-way point , so all of the strangeness after that did start to make a little bit of sense , but it still didn\\'t the make the film all that more entertaining . \\ni guess the bottom line with movies like this is that you should always make sure that the audience is \" into it \" even before they are given the secret password to enter your world of understanding . \\ni mean , showing melissa sagemiller running away from visions for about 20 minutes throughout the movie is just plain lazy ! ! \\nokay , we get it . . . there \\nare people chasing her and we don\\'t know who they are . \\ndo we really need to see it over and over again ? \\nhow about giving us different scenes offering further insight into all of the strangeness going down in the movie ? \\napparently , the studio took this film away from its director and chopped it up themselves , and it shows . \\nthere might\\'ve been a pretty decent teen mind-fuck movie in here somewhere , but i guess \" the suits \" decided that turning it into a music video with little edge , would make more sense . \\nthe actors are pretty good for the most part , although wes bentley just seemed to be playing the exact same character that he did in american beauty , only in a new neighborhood . \\nbut my biggest kudos go out to sagemiller , who holds her own throughout the entire film , and actually has you feeling her character\\'s unraveling . \\noverall , the film doesn\\'t stick because it doesn\\'t entertain , it\\'s confusing , it rarely excites and it feels pretty redundant for most of its runtime , despite a pretty cool ending and explanation to all of the craziness that came before it . \\noh , and by the way , this is not a horror or teen slasher flick . . . it\\'s \\njust packaged to look that way because someone is apparently assuming that the genre is still hot with the kids . \\nit also wrapped production two years ago and has been sitting on the shelves ever since . \\nwhatever . . . skip \\nit ! \\nwhere\\'s joblo coming from ? \\na nightmare of elm street 3 ( 7/10 ) - blair witch 2 ( 7/10 ) - the crow ( 9/10 ) - the crow : salvation ( 4/10 ) - lost highway ( 10/10 ) - memento ( 10/10 ) - the others ( 9/10 ) - stir of echoes ( 8/10 ) \\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def moviews2df(file_ids):\n",
        "    sentiments = [1 if id[:3] == 'pos' else 0 for id in file_ids]\n",
        "    texts = [movie_reviews.raw(id) for id in file_ids]\n",
        "    tokens = [movie_reviews.words(id) for id in file_ids]\n",
        "    df = pd.DataFrame(\n",
        "        {\n",
        "            \"sentiment\": sentiments,\n",
        "            \"text\": texts,\n",
        "            \"tokens\": tokens,\n",
        "        }\n",
        "    )\n",
        "    return df"
      ],
      "metadata": {
        "id": "-yi7T4DrDGvp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "movies = moviews2df(file_ids)\n",
        "movies"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "id": "eRjMjLgpEbGQ",
        "outputId": "faab06b9-6d9a-4a54-da09-3cf1d5b820c7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      sentiment                                               text  \\\n",
              "0             0  plot : two teen couples go to a church party ,...   \n",
              "1             0  the happy bastard's quick movie review \\ndamn ...   \n",
              "2             0  it is movies like these that make a jaded movi...   \n",
              "3             0   \" quest for camelot \" is warner bros . ' firs...   \n",
              "4             0  synopsis : a mentally unstable man undergoing ...   \n",
              "...         ...                                                ...   \n",
              "1995          1  wow ! what a movie . \\nit's everything a movie...   \n",
              "1996          1  richard gere can be a commanding actor , but h...   \n",
              "1997          1  glory--starring matthew broderick , denzel was...   \n",
              "1998          1  steven spielberg's second epic film on world w...   \n",
              "1999          1  truman ( \" true-man \" ) burbank is the perfect...   \n",
              "\n",
              "                                                 tokens  \n",
              "0     (plot, :, two, teen, couples, go, to, a, churc...  \n",
              "1     (the, happy, bastard, ', s, quick, movie, revi...  \n",
              "2     (it, is, movies, like, these, that, make, a, j...  \n",
              "3     (\", quest, for, camelot, \", is, warner, bros, ...  \n",
              "4     (synopsis, :, a, mentally, unstable, man, unde...  \n",
              "...                                                 ...  \n",
              "1995  (wow, !, what, a, movie, ., it, ', s, everythi...  \n",
              "1996  (richard, gere, can, be, a, commanding, actor,...  \n",
              "1997  (glory, --, starring, matthew, broderick, ,, d...  \n",
              "1998  (steven, spielberg, ', s, second, epic, film, ...  \n",
              "1999  (truman, (, \", true, -, man, \", ), burbank, is...  \n",
              "\n",
              "[2000 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2931bc8f-b775-40a5-a0c9-806ab21768c9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "      <th>text</th>\n",
              "      <th>tokens</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>plot : two teen couples go to a church party ,...</td>\n",
              "      <td>(plot, :, two, teen, couples, go, to, a, churc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>the happy bastard's quick movie review \\ndamn ...</td>\n",
              "      <td>(the, happy, bastard, ', s, quick, movie, revi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>it is movies like these that make a jaded movi...</td>\n",
              "      <td>(it, is, movies, like, these, that, make, a, j...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>\" quest for camelot \" is warner bros . ' firs...</td>\n",
              "      <td>(\", quest, for, camelot, \", is, warner, bros, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>synopsis : a mentally unstable man undergoing ...</td>\n",
              "      <td>(synopsis, :, a, mentally, unstable, man, unde...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1995</th>\n",
              "      <td>1</td>\n",
              "      <td>wow ! what a movie . \\nit's everything a movie...</td>\n",
              "      <td>(wow, !, what, a, movie, ., it, ', s, everythi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1996</th>\n",
              "      <td>1</td>\n",
              "      <td>richard gere can be a commanding actor , but h...</td>\n",
              "      <td>(richard, gere, can, be, a, commanding, actor,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1997</th>\n",
              "      <td>1</td>\n",
              "      <td>glory--starring matthew broderick , denzel was...</td>\n",
              "      <td>(glory, --, starring, matthew, broderick, ,, d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1998</th>\n",
              "      <td>1</td>\n",
              "      <td>steven spielberg's second epic film on world w...</td>\n",
              "      <td>(steven, spielberg, ', s, second, epic, film, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1999</th>\n",
              "      <td>1</td>\n",
              "      <td>truman ( \" true-man \" ) burbank is the perfect...</td>\n",
              "      <td>(truman, (, \", true, -, man, \", ), burbank, is...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2000 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2931bc8f-b775-40a5-a0c9-806ab21768c9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2931bc8f-b775-40a5-a0c9-806ab21768c9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2931bc8f-b775-40a5-a0c9-806ab21768c9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Hugging Face Datasets"
      ],
      "metadata": {
        "id": "BK1SdCuGtsGT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets"
      ],
      "metadata": {
        "id": "gQPcWZSd04TB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import list_datasets"
      ],
      "metadata": {
        "id": "LYS86hnk4viR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hf_datasets = list_datasets()\n",
        "print(\"Number of datasets in the Datasets library: \", len(hf_datasets), \"\\n\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nGlKdVSl421W",
        "outputId": "08457095-fae8-4649-98f7-4c138afef101"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of datasets in the Datasets library:  4307 \n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hf_datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p6yQ1ExR2jw4",
        "outputId": "1374690c-73db-4cb3-eabd-1d8bdaa719ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['assin',\n",
              " 'ar_res_reviews',\n",
              " 'ambig_qa',\n",
              " 'bianet',\n",
              " 'ag_news',\n",
              " 'ajgt_twitter_ar',\n",
              " 'aeslc',\n",
              " 'bc2gm_corpus',\n",
              " 'air_dialogue',\n",
              " 'acronym_identification',\n",
              " 'afrikaans_ner_corpus',\n",
              " 'allegro_reviews',\n",
              " 'ade_corpus_v2',\n",
              " 'adversarial_qa',\n",
              " 'alt',\n",
              " 'billsum',\n",
              " 'amazon_polarity',\n",
              " 'amttl',\n",
              " 'ascent_kb',\n",
              " 'big_patent',\n",
              " 'bn_hate_speech',\n",
              " 'bswac',\n",
              " 'capes',\n",
              " 'arsentd_lev',\n",
              " 'bible_para',\n",
              " 'amazon_reviews_multi',\n",
              " 'ai2_arc',\n",
              " 'beans',\n",
              " 'anli',\n",
              " 'arabic_pos_dialect',\n",
              " 'best2009',\n",
              " 'boolq',\n",
              " 'ar_sarcasm',\n",
              " 'asnq',\n",
              " 'bnl_newspapers',\n",
              " 'amazon_us_reviews',\n",
              " 'arxiv_dataset',\n",
              " 'banking77',\n",
              " 'bookcorpus',\n",
              " 'bprec',\n",
              " 'c3',\n",
              " 'allocine',\n",
              " 'aslg_pc12',\n",
              " 'biosses',\n",
              " 'bbc_hindi_nli',\n",
              " 'americas_nli',\n",
              " 'biomrc',\n",
              " 'blimp',\n",
              " 'aqua_rat',\n",
              " 'aquamuse',\n",
              " 'ar_cov19',\n",
              " 'arabic_speech_corpus',\n",
              " 'arcd',\n",
              " 'blended_skill_talk',\n",
              " 'cail2018',\n",
              " 'blbooksgenre',\n",
              " 'bookcorpusopen',\n",
              " 'bsd_ja_en',\n",
              " 'atomic',\n",
              " 'autshumato',\n",
              " 'blog_authorship_corpus',\n",
              " 'caner',\n",
              " 'app_reviews',\n",
              " 'arabic_billion_words',\n",
              " 'art',\n",
              " 'asset',\n",
              " 'blbooks',\n",
              " 'brwac',\n",
              " 'c4',\n",
              " 'ami',\n",
              " 'assin2',\n",
              " 'babi_qa',\n",
              " 'bbaw_egyptian',\n",
              " 'bing_coronavirus_query_set',\n",
              " 'break_data',\n",
              " 'chr_en',\n",
              " 'covid_qa_deepset',\n",
              " 'cifar10',\n",
              " 'cifar100',\n",
              " 'cdsc',\n",
              " 'catalonia_independence',\n",
              " 'circa',\n",
              " 'cc100',\n",
              " 'cawac',\n",
              " 'clue',\n",
              " 'cdt',\n",
              " 'codah',\n",
              " 'casino',\n",
              " 'clickbait_news_bg',\n",
              " 'cc_news',\n",
              " 'ccaligned_multilingual',\n",
              " 'cedr',\n",
              " 'code_x_glue_cc_cloze_testing_maxmin',\n",
              " 'code_x_glue_cc_code_refinement',\n",
              " 'conv_ai',\n",
              " 'conv_ai_3',\n",
              " 'conv_questions',\n",
              " 'covost2',\n",
              " 'cmrc2018',\n",
              " 'commonsense_qa',\n",
              " 'code_x_glue_ct_code_to_text',\n",
              " 'conceptnet5',\n",
              " 'cord19',\n",
              " 'cnn_dailymail',\n",
              " 'code_x_glue_cc_code_to_code_trans',\n",
              " 'com_qa',\n",
              " 'covid_tweets_japanese',\n",
              " 'civil_comments',\n",
              " 'code_x_glue_tc_text_to_code',\n",
              " 'coqa',\n",
              " 'cosmos_qa',\n",
              " 'cfq',\n",
              " 'code_x_glue_cc_clone_detection_poj104',\n",
              " 'cos_e',\n",
              " 'craigslist_bargains',\n",
              " 'cats_vs_dogs',\n",
              " 'common_voice',\n",
              " 'cbt',\n",
              " 'code_search_net',\n",
              " 'climate_fever',\n",
              " 'code_x_glue_cc_clone_detection_big_clone_bench',\n",
              " 'clinc_oos',\n",
              " 'code_x_glue_tc_nl_code_search_adv',\n",
              " 'cmu_hinglish_dog',\n",
              " 'code_x_glue_tt_text_to_text',\n",
              " 'coached_conv_pref',\n",
              " 'cppe-5',\n",
              " 'cornell_movie_dialog',\n",
              " 'coarse_discourse',\n",
              " 'code_x_glue_cc_cloze_testing_all',\n",
              " 'code_x_glue_cc_code_completion_token',\n",
              " 'code_x_glue_cc_defect_detection',\n",
              " 'competition_math',\n",
              " 'conllpp',\n",
              " 'covid_qa_castorini',\n",
              " 'common_gen',\n",
              " 'common_language',\n",
              " 'compguesswhat',\n",
              " 'conll2003',\n",
              " 'code_x_glue_cc_code_completion_line',\n",
              " 'consumer-finance-complaints',\n",
              " 'conv_ai_2',\n",
              " 'counter',\n",
              " 'conll2000',\n",
              " 'conll2002',\n",
              " 'covid_qa_ucsd',\n",
              " 'crawl_domain',\n",
              " 'dbrd',\n",
              " 'dart',\n",
              " 'daily_dialog',\n",
              " 'curiosity_dialogs',\n",
              " 'cryptonite',\n",
              " 'crd3',\n",
              " 'fashion_mnist',\n",
              " 'danish_political_comments',\n",
              " 'ecb',\n",
              " 'docred',\n",
              " 'doc2dial',\n",
              " 'crime_and_punish',\n",
              " 'crows_pairs',\n",
              " 'dane',\n",
              " 'electricity_load_diagrams',\n",
              " 'eli5',\n",
              " 'empathetic_dialogues',\n",
              " 'europa_ecdc_tm',\n",
              " 'cuad',\n",
              " 'datacommons_factcheck',\n",
              " 'definite_pronoun_resolution',\n",
              " 'dream',\n",
              " 'emo',\n",
              " 'ehealth_kd',\n",
              " 'emotone_ar',\n",
              " 'fquad',\n",
              " 'eli5_category',\n",
              " 'europarl_bilingual',\n",
              " 'fake_news_english',\n",
              " 'gem',\n",
              " 'discovery',\n",
              " 'dialog_re',\n",
              " 'dutch_social',\n",
              " 'emea',\n",
              " 'enriched_web_nlg',\n",
              " 'factckbr',\n",
              " 'finer',\n",
              " 'food101',\n",
              " 'duorc',\n",
              " 'e2e_nlg',\n",
              " 'eu_regulatory_ir',\n",
              " 'event2Mind',\n",
              " 'exams',\n",
              " 'fever',\n",
              " 'deal_or_no_dialog',\n",
              " 'cs_restaurants',\n",
              " 'disfl_qa',\n",
              " 'esnli',\n",
              " 'dyk',\n",
              " 'eth_py150_open',\n",
              " 'ecthr_cases',\n",
              " 'euronews',\n",
              " 'evidence_infer_treatment',\n",
              " 'flores',\n",
              " 'flue',\n",
              " 'eduge',\n",
              " 'emotion',\n",
              " 'eraser_multi_rc',\n",
              " 'farsi_news',\n",
              " 'gap',\n",
              " 'dengue_filipino',\n",
              " 'diplomacy_detection',\n",
              " 'drop',\n",
              " 'e2e_nlg_cleaned',\n",
              " 'dbpedia_14',\n",
              " 'disaster_response_messages',\n",
              " 'eitb_parcc',\n",
              " 'discofuse',\n",
              " 'ethos',\n",
              " 'doqa',\n",
              " 'fake_news_filipino',\n",
              " 'eurlex',\n",
              " 'freebase_qa',\n",
              " 'few_rel',\n",
              " 'financial_phrasebank',\n",
              " 'europa_eac_tm',\n",
              " 'harem',\n",
              " 'gigaword',\n",
              " 'glue',\n",
              " 'hatexplain',\n",
              " 'hellaswag',\n",
              " 'has_part',\n",
              " 'great_code',\n",
              " 'gutenberg_time',\n",
              " 'generics_kb',\n",
              " 'giga_fren',\n",
              " 'go_emotions',\n",
              " 'generated_reviews_enth',\n",
              " 'gooaq',\n",
              " 'hate_speech_offensive',\n",
              " 'german_legal_entity_recognition',\n",
              " 'glucose',\n",
              " 'jigsaw_unintended_bias',\n",
              " 'lama',\n",
              " 'mbpp',\n",
              " 'md_gender_bias',\n",
              " 'med_hop',\n",
              " 'ms_terms',\n",
              " 'newspop',\n",
              " 'pass',\n",
              " 'proto_qa',\n",
              " 'ptb_text_only',\n",
              " 'ronec',\n",
              " 'samsum',\n",
              " 'sanskrit_classic',\n",
              " 'scan',\n",
              " 'sepedi_ner',\n",
              " 'social_bias_frames',\n",
              " 'spider',\n",
              " 'tashkeela',\n",
              " 'time_dial',\n",
              " 'germaner',\n",
              " 'germeval_14',\n",
              " 'hate_speech_filipino',\n",
              " 'hausa_voa_ner',\n",
              " 'hebrew_this_world',\n",
              " 'id_nergrit_corpus',\n",
              " 'inquisitive_qg',\n",
              " 'kilt_tasks',\n",
              " 'kor_ner',\n",
              " 'labr',\n",
              " 'laroseda',\n",
              " 'lj_speech',\n",
              " 'math_qa',\n",
              " 'multi_nli',\n",
              " 'mutual_friends',\n",
              " 'mwsc',\n",
              " 'narrativeqa_manual',\n",
              " 'natural_questions',\n",
              " 'nkjp-ner',\n",
              " 'norne',\n",
              " 'numer_sense',\n",
              " 'ollie',\n",
              " 'onestop_qa',\n",
              " 'opus_ubuntu',\n",
              " 'para_crawl',\n",
              " 'per_sent',\n",
              " 'polyglot_ner',\n",
              " 'quora',\n",
              " 'riddle_sense',\n",
              " 'smartdata',\n",
              " 'snips_built_in_intents',\n",
              " 'squad_it',\n",
              " 'ted_iwlst2013',\n",
              " 'thaiqa_squad',\n",
              " 'google_wellformed_query',\n",
              " 'hate_speech_portuguese',\n",
              " 'id_newspapers_2018',\n",
              " 'hotpot_qa',\n",
              " 'id_panl_bppt',\n",
              " 'interpress_news_category_tr',\n",
              " 'kan_hope',\n",
              " 'journalists_questions',\n",
              " 'kor_sae',\n",
              " 'lince',\n",
              " 'masakhaner',\n",
              " 'm_lama',\n",
              " 'mocha',\n",
              " 'medal',\n",
              " 'newsph',\n",
              " 'mkb',\n",
              " 'opus_montenegrinsubs',\n",
              " 'muchocine',\n",
              " 'parsinlu_reading_comprehension',\n",
              " 'myanmar_news',\n",
              " 'paws-x',\n",
              " 'ncbi_disease',\n",
              " 'ro_sts',\n",
              " 'norwegian_ner',\n",
              " 'scielo',\n",
              " 'opus100',\n",
              " 'scitldr',\n",
              " 'opus_books',\n",
              " 'senti_ws',\n",
              " 'sharc',\n",
              " 'opus_finlex',\n",
              " 'orange_sum',\n",
              " 'sick',\n",
              " 'piaf',\n",
              " 'tatoeba',\n",
              " 'qa_zre',\n",
              " 'the_pile',\n",
              " 'qangaroo',\n",
              " 'tilde_model',\n",
              " 'sem_eval_2014_task_1',\n",
              " 'times_of_india_news_headlines',\n",
              " 'sharc_modified',\n",
              " 'siswati_ner_corpus',\n",
              " 'sms_spam',\n",
              " 'sofc_materials_articles',\n",
              " 'spc',\n",
              " 'sst',\n",
              " 'stsb_mt_sv',\n",
              " 'swda',\n",
              " 'swedish_medical_ner',\n",
              " 'hate_speech_pl',\n",
              " 'hausa_voa_topics',\n",
              " 'hope_edi',\n",
              " 'imdb_urdu_reviews',\n",
              " 'jfleg',\n",
              " 'kor_3i4k',\n",
              " 'kor_nli',\n",
              " 'lm1b',\n",
              " 'mac_morpho',\n",
              " 'mdd',\n",
              " 'msr_zhen_translation_parity',\n",
              " 'newsroom',\n",
              " 'numeric_fused_head',\n",
              " 'openslr',\n",
              " 'pec',\n",
              " 'poleval2019_mt',\n",
              " 'py_ast',\n",
              " 'qed_amara',\n",
              " 'refresd',\n",
              " 'squadshifts',\n",
              " 'superb',\n",
              " 'svhn',\n",
              " 'swag',\n",
              " 'tab_fact',\n",
              " 'tamilmixsentiment',\n",
              " 'text2log',\n",
              " 'the_pile_books3',\n",
              " 'humicroedit',\n",
              " 'irc_disentangle',\n",
              " 'klue',\n",
              " 'lener_br',\n",
              " 'liar',\n",
              " 'liveqa',\n",
              " 'menyo20k_mt',\n",
              " 'movie_rationales',\n",
              " 'opus_fiskmo',\n",
              " 'peer_read',\n",
              " 'pib',\n",
              " 'qasc',\n",
              " 'quac',\n",
              " 'ro_sent',\n",
              " 's2orc',\n",
              " 'sem_eval_2018_task_1',\n",
              " 'sogou_news',\n",
              " 'spanish_billion_words',\n",
              " 'super_glue',\n",
              " 'ted_multi',\n",
              " 'hansards',\n",
              " 'hard',\n",
              " 'hate_offensive',\n",
              " 'indic_glue',\n",
              " 'kor_nlu',\n",
              " 'makhzan',\n",
              " 'math_dataset',\n",
              " 'matinf',\n",
              " 'mlsum',\n",
              " 'msra_ner',\n",
              " 'nlu_evaluation_data',\n",
              " 'ohsumed',\n",
              " 'openwebtext',\n",
              " 'opus_elhuyar',\n",
              " 'persian_ner',\n",
              " 'poem_sentiment',\n",
              " 'prachathai67k',\n",
              " 'psc',\n",
              " 'pubmed_qa',\n",
              " 'quartz',\n",
              " 're_dial',\n",
              " 'rotten_tomatoes',\n",
              " 'scene_parse_150',\n",
              " 'schema_guided_dstc8',\n",
              " 'sciq',\n",
              " 'scitail',\n",
              " 'swahili',\n",
              " 'thai_toxicity_tweet',\n",
              " 'the_pile_openwebtext2',\n",
              " 'hendrycks_test',\n",
              " 'hippocorpus',\n",
              " 'hrwac',\n",
              " 'hyperpartisan_news_detection',\n",
              " 'imppres',\n",
              " 'large_spanish_corpus',\n",
              " 'multi_booked',\n",
              " 'opus_paracrawl',\n",
              " 'paws',\n",
              " 'pn_summary',\n",
              " 'quail',\n",
              " 'red_caps',\n",
              " 'ropes',\n",
              " 'simple_questions_v2',\n",
              " 'squad_adversarial',\n",
              " 'squad_es',\n",
              " 'squad_kor_v2',\n",
              " 'story_cloze',\n",
              " 'swahili_news',\n",
              " 'taskmaster1',\n",
              " 'tep_en_fa_para',\n",
              " 'hindi_discourse',\n",
              " 'gnad10',\n",
              " 'hda_nli_hindi',\n",
              " 'imdb',\n",
              " 'hebrew_sentiment',\n",
              " 'lambada',\n",
              " 'id_clickbait',\n",
              " 'lc_quad',\n",
              " 'id_puisi',\n",
              " 'lst20',\n",
              " 'isixhosa_ner_corpus',\n",
              " 'iwslt2017',\n",
              " 'mc4',\n",
              " 'jnlpba',\n",
              " 'kde4',\n",
              " 'multi_eurlex',\n",
              " 'multi_para_crawl',\n",
              " 'kor_hate',\n",
              " 'multi_re_qa',\n",
              " 'kor_qpair',\n",
              " 'multi_woz_v22',\n",
              " 'linnaeus',\n",
              " 'oclar',\n",
              " 'mrqa',\n",
              " 'offenseval_dravidian',\n",
              " 'msr_text_compression',\n",
              " 'ofis_publik',\n",
              " 'multilingual_librispeech',\n",
              " 'omp',\n",
              " 'neural_code_search',\n",
              " 'openai_humaneval',\n",
              " 'offcombr',\n",
              " 'php',\n",
              " 'opus_dogc',\n",
              " 'reasoning_bg',\n",
              " 'poleval2019_cyberbullying',\n",
              " 'scicite',\n",
              " 'qasper',\n",
              " 'sem_eval_2010_task_8',\n",
              " 'race',\n",
              " 'sem_eval_2020_task_11',\n",
              " 'scifact',\n",
              " 'sesotho_ner_corpus',\n",
              " 'setimes',\n",
              " 'setswana_ner_corpus',\n",
              " 'species_800',\n",
              " 'style_change_detection',\n",
              " 'subjqa',\n",
              " 'tanzil',\n",
              " 'taskmaster2',\n",
              " 'the_pile_stack_exchange',\n",
              " 'hlgd',\n",
              " 'greek_legal_code',\n",
              " 'hrenwac_para',\n",
              " 'iapp_wiki_qa_squad',\n",
              " 'guardian_authorship',\n",
              " 'id_liputan6',\n",
              " 'hate_speech18',\n",
              " 'head_qa',\n",
              " 'igbo_english_machine_translation',\n",
              " 'hover',\n",
              " 'isizulu_ner_corpus',\n",
              " 'jeopardy',\n",
              " 'kannada_news',\n",
              " 'ms_marco',\n",
              " 'limit',\n",
              " 'msr_genomics_kbcomp',\n",
              " 'mc_taco',\n",
              " 'mt_eng_vietnamese',\n",
              " 'norec',\n",
              " 'multi_news',\n",
              " 'opus_dgt',\n",
              " 'multi_nli_mismatch',\n",
              " 'opus_infopankki',\n",
              " 'multi_x_science_sum',\n",
              " 'polemo2',\n",
              " 'narrativeqa',\n",
              " 'quarel',\n",
              " 'quoref',\n",
              " 'ncslgr',\n",
              " 'reddit_tifu',\n",
              " 'openbookqa',\n",
              " 'roman_urdu',\n",
              " 'opus_euconst',\n",
              " 'peoples_daily_ner',\n",
              " 'silicone',\n",
              " 'polsum',\n",
              " 'snow_simplified_japanese_corpus',\n",
              " 'so_stacksample',\n",
              " 'pubmed',\n",
              " 'social_i_qa',\n",
              " 'reuters21578',\n",
              " 'saudinewsnet',\n",
              " 'stsb_multi_mt',\n",
              " 'thaisum',\n",
              " 'sent_comp',\n",
              " 'squad_v2',\n",
              " 'srwac',\n",
              " 'tapaco',\n",
              " 'taskmaster3',\n",
              " 'ted_talks_iwslt',\n",
              " 'health_fact',\n",
              " 'ilist',\n",
              " 'kd_conv',\n",
              " 'librispeech_asr',\n",
              " 'librispeech_lm',\n",
              " 'metrec',\n",
              " 'mkqa',\n",
              " 'mlqa',\n",
              " 'multidoc2dial',\n",
              " 'news_commentary',\n",
              " 'newsph_nli',\n",
              " 'opus_rf',\n",
              " 'opus_tedtalks',\n",
              " 'opus_wikipedia',\n",
              " 'oscar',\n",
              " 'pg19',\n",
              " 'qanta',\n",
              " 'russian_super_glue',\n",
              " 'senti_lex',\n",
              " 'sentiment140',\n",
              " 'squad',\n",
              " 'squad_v1_pt',\n",
              " 'stereoset',\n",
              " 'swiss_judgment_prediction',\n",
              " 'telugu_books',\n",
              " 'hind_encorp',\n",
              " 'hkcancor',\n",
              " 'hybrid_qa',\n",
              " 'indonli',\n",
              " 'kilt_wikipedia',\n",
              " 'kor_sarcasm',\n",
              " 'medical_questions_pairs',\n",
              " 'moroco',\n",
              " 'nell',\n",
              " 'newsgroup',\n",
              " 'nli_tr',\n",
              " 'offenseval2020_tr',\n",
              " 'onestop_english',\n",
              " 'open_subtitles',\n",
              " 'opinosis',\n",
              " 'opus_gnome',\n",
              " 'para_pat',\n",
              " 'qa_srl',\n",
              " 'reclor',\n",
              " 'reddit',\n",
              " 'sberquad',\n",
              " 'scb_mt_enth_2020',\n",
              " 'search_qa',\n",
              " 'sede',\n",
              " 'snli',\n",
              " 'speech_commands',\n",
              " 'swedish_ner_corpus',\n",
              " 'swedish_reviews',\n",
              " 'telugu_news',\n",
              " 'hans',\n",
              " 'igbo_ner',\n",
              " 'interpress_news_category_tr_lite',\n",
              " 'jigsaw_toxicity_pred',\n",
              " 'kinnews_kirnews',\n",
              " 'lex_glue',\n",
              " 'medical_dialog',\n",
              " 'miam',\n",
              " 'msr_sqa',\n",
              " 'nsmc',\n",
              " 'opus_memat',\n",
              " 'opus_openoffice',\n",
              " 'piqa',\n",
              " 'pragmeval',\n",
              " 'qa4mre',\n",
              " 'qed',\n",
              " 'selqa',\n",
              " 'squad_kor_v1',\n",
              " 'thainer',\n",
              " 'grail_qa',\n",
              " 'hebrew_projectbenyehuda',\n",
              " 'igbo_monolingual',\n",
              " 'indonlu',\n",
              " 'kelm',\n",
              " 'meta_woz',\n",
              " 'metooma',\n",
              " 'mnist',\n",
              " 'nchlt',\n",
              " 'newsqa',\n",
              " 'nq_open',\n",
              " 'opus_xhosanavy',\n",
              " 'recipe_nlg',\n",
              " 'ro_sts_parallel',\n",
              " 'scientific_papers',\n",
              " 'ted_hrlr',\n",
              " 'timit_asr',\n",
              " 'un_ga',\n",
              " 'totto',\n",
              " 'tlc',\n",
              " 'wiki_hop',\n",
              " 'trec',\n",
              " 'tweets_ar_en_parallel',\n",
              " 'told-br',\n",
              " 'tmu_gfm_dataset',\n",
              " 'udhr',\n",
              " 'tsac',\n",
              " 'ubuntu_dialogs_corpus',\n",
              " 'tuple_ie',\n",
              " 'turku_ner_corpus',\n",
              " 'urdu_fake_news',\n",
              " 'wiki_bio',\n",
              " 'wiki_source',\n",
              " 'wikiann',\n",
              " 'wili_2018',\n",
              " 'wmt18',\n",
              " 'x_stance',\n",
              " 'trivia_qa',\n",
              " 'ttc4900',\n",
              " 'twi_wordsim353',\n",
              " 'wiki_split',\n",
              " 'wmt16',\n",
              " 'xglue',\n",
              " 'xquad',\n",
              " 'xsum',\n",
              " 'vivos',\n",
              " 'wiki_movies',\n",
              " 'wikihow',\n",
              " 'wisesight_sentiment',\n",
              " 'xquad_r',\n",
              " 'yoruba_bbc_topics',\n",
              " 'turkish_ner',\n",
              " 'tweet_qa',\n",
              " 'wiki_qa',\n",
              " 'wisesight1000',\n",
              " 'wmt_t2t',\n",
              " 'wnut_17',\n",
              " 'wrbsc',\n",
              " 'xcsr',\n",
              " 'xnli',\n",
              " 'turk',\n",
              " 'twi_text_c3',\n",
              " 'un_pc',\n",
              " 'universal_dependencies',\n",
              " 'weibo_ner',\n",
              " 'wiki_qa_ar',\n",
              " 'wikipedia',\n",
              " 'tiny_shakespeare',\n",
              " 'wiki_auto',\n",
              " 'wiki_lingua',\n",
              " 'wikisql',\n",
              " 'wmt14',\n",
              " 'wmt20_mlqe_task3',\n",
              " 'turkish_shrinked_ner',\n",
              " 'tydiqa',\n",
              " 'universal_morphologies',\n",
              " 'wikicorpus',\n",
              " 'wikitext',\n",
              " 'turkish_movie_sentiment',\n",
              " 'turkish_product_reviews',\n",
              " 'tweet_eval',\n",
              " 'wiki_snippets',\n",
              " 'wino_bias',\n",
              " 'winogrande',\n",
              " 'wongnai_reviews',\n",
              " 'woz_dialogue',\n",
              " 'xcopa',\n",
              " 'yelp_polarity',\n",
              " 'zest',\n",
              " 'um005',\n",
              " 'urdu_sentiment_corpus',\n",
              " 'web_of_science',\n",
              " 'winograd_wsc',\n",
              " 'wmt17',\n",
              " 'yahoo_answers_topics',\n",
              " 'turkic_xwmt',\n",
              " 'tweets_hate_speech_detection',\n",
              " 'vctk',\n",
              " 'wiki40b',\n",
              " 'wiki_asp',\n",
              " 'wmt15',\n",
              " 'wmt20_mlqe_task2',\n",
              " 'xor_tydi_qa',\n",
              " 'xsum_factuality',\n",
              " 'xtreme',\n",
              " 'yahoo_answers_qa',\n",
              " 'wiki_dpr',\n",
              " 'wiqa',\n",
              " 'yoruba_text_c3',\n",
              " 'yoruba_wordsim353',\n",
              " 'tunizi',\n",
              " 'web_nlg',\n",
              " 'wider_face',\n",
              " 'wiki_atomic_edits',\n",
              " 'wiki_summary',\n",
              " 'xed_en_fi',\n",
              " 'youtube_caption_corrections',\n",
              " 'wi_locness',\n",
              " 'wikitext_tl39',\n",
              " 'wmt20_mlqe_task1',\n",
              " 'yelp_review_full',\n",
              " 'un_multi',\n",
              " 'web_questions',\n",
              " 'wmt19',\n",
              " 'yoruba_gv_ner',\n",
              " 'AConsApart/anime_subtitles_DialoGPT',\n",
              " 'AI-it/korean-hate-speech',\n",
              " 'Abirate/french_book_reviews',\n",
              " 'ARTeLab/fanpage',\n",
              " '0n1xus/pytorrent-standalone',\n",
              " '0n1xus/codexglue',\n",
              " 'ASCCCCCCCC/amazon_zh_simple',\n",
              " 'AlgoveraAI/CryptoPunks',\n",
              " 'Adnan/Urdu_News_Headlines',\n",
              " 'ARKseal/YFCC14M_subset_webdataset',\n",
              " 'Abdo1Kamr/Arabic_Hadith',\n",
              " 'Akshith/aa',\n",
              " 'AndrewMcDowell/de_corpora_parliament_processed',\n",
              " 'AHussain0418/day2_data',\n",
              " 'AlekseyKorshuk/horror-scripts',\n",
              " 'AI-Sweden/SuperLim',\n",
              " 'AI-it/khs_service_test',\n",
              " 'ARTeLab/mlsum-it',\n",
              " 'ARTeLab/ilpost',\n",
              " 'Abirate/code_net_dev_dataset',\n",
              " 'AhmadSawal/qa',\n",
              " 'Akshith/g_rock',\n",
              " 'Aisha/BAAD16',\n",
              " 'AhmedSSoliman/CoNaLa',\n",
              " 'Aisha/BAAD6',\n",
              " 'AlexMaclean/wikipedia-deletion-compressions',\n",
              " 'AlexZapolskii/zapolskii-amazon',\n",
              " 'ASCCCCCCCC/amazon_zh',\n",
              " 'Akila/ForgottenRealmsWikiDataset',\n",
              " 'AHussain0418/day4data',\n",
              " 'AdWeeb/DravidianMT',\n",
              " 'AlekseyDorkin/extended_tweet_emojis',\n",
              " 'Aliseyfi/event_token_type',\n",
              " 'Akshith/test',\n",
              " 'Alvenir/nst-da-16khz',\n",
              " 'AHussain0418/demo_data',\n",
              " 'Abirate/english_quotes',\n",
              " 'Abirate/code_net_dataset',\n",
              " 'Abirate/code_net_test_final_dataset',\n",
              " 'AlexMaclean/all-deletion-compressions',\n",
              " 'AlekseyKorshuk/comedy-scripts',\n",
              " 'Annabelleabbott/real-fake-news-workshop',\n",
              " 'Anurag-Singh-creator/tasks',\n",
              " 'BSC-TeMU/ancora-ca-ner',\n",
              " 'Binbin/my_dataset',\n",
              " 'DDSC/europarl',\n",
              " 'DanL/scientific-challenges-and-directions-dataset',\n",
              " 'Datatang/accented_english',\n",
              " 'DoyyingFace/github-embeddings-doy',\n",
              " 'GEM/CrossWOZ',\n",
              " 'Atsushi/fungi_diagnostic_chars_comparison_japanese',\n",
              " 'Cyberfish/text_error_correction',\n",
              " 'Datatang/multi_language',\n",
              " 'Emma121/testtest',\n",
              " 'Enes3774/data',\n",
              " 'FRTNX/cosuju',\n",
              " 'CodedotAI/code_clippy',\n",
              " 'DDSC/reddit-da-asr-preprocessed',\n",
              " 'DeskDown/ALTDataset',\n",
              " 'Anurag-Singh-creator/task',\n",
              " 'Baybars/parla_text_corpus',\n",
              " 'Bosio/pacman_descriptions',\n",
              " 'Check/region_7',\n",
              " 'Check/region_9',\n",
              " 'Check/vverify',\n",
              " 'Cyberfish/pos_tagger',\n",
              " 'Chuu/Vhh',\n",
              " 'Exr0n/wiki-entity-similarity',\n",
              " 'DelgadoPanadero/Pokemon',\n",
              " 'DiFronzo/Human_Activity_Recognition',\n",
              " 'ESZER/H',\n",
              " 'Firoj/CrisisBench',\n",
              " 'Fraser/python-state-changes',\n",
              " 'Arnold/hausa_common_voice',\n",
              " 'BSC-TeMU/sts-ca',\n",
              " 'BeIR/beir',\n",
              " 'Annielytics/DoctorsNotes',\n",
              " 'DrishtiSharma/as_opus100_processed',\n",
              " 'DrishtiSharma/br_opus100_processed',\n",
              " 'EMBO/biolang',\n",
              " 'Finnish-NLP/mc4_fi_cleaned',\n",
              " 'Francois/futures_es',\n",
              " 'BSC-TeMU/viquiquad',\n",
              " 'BritishLibraryLabs/EThOS-PhD-metadata',\n",
              " 'Check/region_8',\n",
              " 'BeIR/beir-corpus',\n",
              " 'Crives/haha',\n",
              " 'BSC-TeMU/SQAC',\n",
              " 'Check/a_re_gi',\n",
              " 'Chun/dataset',\n",
              " 'DDSC/dkhate',\n",
              " 'DDSC/partial-danish-gigaword-no-twitter',\n",
              " 'DSCI511G1/COP26_Energy_Transition_Tweets',\n",
              " 'Daniele/dante-corpus',\n",
              " 'Darren/data',\n",
              " 'Datatang/mixed_speech_chinese_english',\n",
              " 'DDSC/angry-tweets',\n",
              " 'Check/region_3',\n",
              " 'DDSC/reddit-da',\n",
              " 'Check/region_6',\n",
              " 'ChristophSchuhmann/MS_COCO_2017_URL_TEXT',\n",
              " 'Davlan/masakhanerV1',\n",
              " 'Doohae/modern_music_re',\n",
              " 'CyranoB/polarity',\n",
              " 'DrishtiSharma/kk_opus100_processed',\n",
              " 'Dumiiii/common-voice-romaniarss',\n",
              " 'DeskDown/ALTDataset_en-to-fil-vi-id-ms-ja-khm',\n",
              " 'DoyyingFace/github-issues-doy',\n",
              " 'Fraser/mnist-text-no-spaces',\n",
              " 'Fraser/mnist-text-small',\n",
              " 'Fraser/short-jokes',\n",
              " 'GEM/ART',\n",
              " 'Emanuel/UD_Portuguese-Bosque',\n",
              " 'Fraser/mnist-text-default',\n",
              " 'Fraser/python-lines',\n",
              " 'BenjaminGalliot/pangloss',\n",
              " 'CAGER/rick',\n",
              " 'CALM/arwiki',\n",
              " 'Check/region_1',\n",
              " 'CodedotAI/code_clippy_github',\n",
              " 'Cropinky/flatearther',\n",
              " 'Datatang/multi_language_conversation',\n",
              " 'DoctorSlimm/yipee',\n",
              " 'FRTNX/worldbank-projects',\n",
              " 'Fraser/wiki_sentences',\n",
              " 'BlakesOrb6/Fred-Flintstone',\n",
              " 'CShorten/KerasBERT',\n",
              " 'Davlan/conll2003_de_noMISC',\n",
              " 'Dmitriy612/1',\n",
              " 'DrishtiSharma/bg_opus100_processed',\n",
              " 'DrishtiSharma/hi_opus100_processed',\n",
              " 'DrishtiSharma/or_opus100_processed',\n",
              " 'Eymen3455/xsum_tr',\n",
              " 'GEM/OrangeSum',\n",
              " 'ApiInferenceTest/asr_dummy',\n",
              " 'CShorten/ZillowPrize',\n",
              " 'CodedotAI/code-clippy-tfrecords',\n",
              " 'Cropinky/rap_lyrics_english',\n",
              " 'DDSC/twitter-sent',\n",
              " 'Davlan/conll2003_noMISC',\n",
              " 'Atsushi/fungi_trait_circus_database',\n",
              " 'AryanLala/autonlp-data-Scientific_Title_Generator',\n",
              " 'Atsushi/fungi_indexed_mycological_papers_japanese',\n",
              " 'BSC-TeMU/tecla',\n",
              " 'Babelscape/wikineural',\n",
              " 'BatuhanYilmaz/github-issues',\n",
              " 'Check/region_2',\n",
              " 'Check/region_4',\n",
              " 'Check/regions',\n",
              " 'Cropinky/wow_fishing_bobber',\n",
              " 'Datatang/accented_mandarin',\n",
              " 'DELith/github-issues',\n",
              " 'Datatang/mandarin_chinese',\n",
              " 'DrishtiSharma/sl_opus100_processed',\n",
              " 'Doohae/klue-mrc-bm25',\n",
              " 'Emon/sobuj',\n",
              " 'Fraser/program-synthesis',\n",
              " 'Bosio/pacman',\n",
              " 'CAiRE/ASCEND',\n",
              " 'Champion/vpc2020_clear_anon_speech',\n",
              " 'Check/region_5',\n",
              " 'Cheranga/test',\n",
              " 'Datatang/chinese_dialect',\n",
              " 'DrishtiSharma/mr_opus100_processed',\n",
              " 'FIG-Loneliness/FIG-Loneliness',\n",
              " 'Felix-ML/quoteli3',\n",
              " 'Fraser/news-category-dataset',\n",
              " 'Avishekavi/Avi',\n",
              " 'BSC-TeMU/xquad-ca',\n",
              " 'Babelscape/rebel-dataset',\n",
              " 'ChadxxxxHall/Inter-vision',\n",
              " 'DDSC/lcc',\n",
              " 'DrishtiSharma/sr_opus100_processed',\n",
              " 'EMBO/sd-nlp',\n",
              " 'FL33TW00D/test-dataset',\n",
              " 'GEM/BiSECT',\n",
              " 'GEM/conversational_weather',\n",
              " 'GEM/e2e_nlg',\n",
              " 'GEM/cochrane-simplification',\n",
              " 'GEM/common_gen',\n",
              " 'GEM/dart',\n",
              " 'GEM/totto',\n",
              " 'GEM/wiki_lingua',\n",
              " 'GEM/Taskmaster',\n",
              " 'GEM/RiSAWOZ',\n",
              " 'GEM/surface_realisation_st_2020',\n",
              " 'GEM/turku_hockey_data2text',\n",
              " 'GEM/indonlg',\n",
              " 'GEM/SIMPITIKI',\n",
              " 'GEM/schema_guided_dialog',\n",
              " 'GEM/dstc10_track2_task2',\n",
              " 'GalacticAI/Noirset',\n",
              " 'Graphcore/vqa',\n",
              " 'HarleyQ/WitcherDialogue',\n",
              " 'HenryAI/KerasCodeExamples.txt',\n",
              " 'Jikiwa/demo3',\n",
              " 'Jikiwa/test-16344349440339',\n",
              " 'Jikiwa/test-16344361893586',\n",
              " 'KBLab/overlim',\n",
              " 'GEM/cs_restaurants',\n",
              " 'Graphcore/gqa-lxmert',\n",
              " 'Graphcore/wikipedia-bert-128',\n",
              " 'HenryAI/KerasAPIReference.txt',\n",
              " 'HenryAI/KerasDeveloperGuides.txt',\n",
              " 'Jack0508/demo',\n",
              " 'Jikiwa/temp-repo-valid',\n",
              " 'KETI-AIR/kor_corpora',\n",
              " 'GEM/viggo',\n",
              " 'GEM/xlsum',\n",
              " 'GEM-submissions/lewtun__hugging-face-test-t5-base.outputs.json-36bf2a59__1645559101',\n",
              " 'GroNLP/ik-nlp-22_slp',\n",
              " 'HHousen/ParaSCI',\n",
              " 'HUPD/hupd',\n",
              " 'Harveenchadha/indic-voice',\n",
              " 'HarveyBWest/mybot',\n",
              " 'Jikiwa/stargazers',\n",
              " 'Jikiwa/test-16344368182003',\n",
              " 'Karavet/ARPA-Armenian-Paraphrase-Corpus',\n",
              " 'GEM/mlb_data_to_text',\n",
              " 'GonzaloA/fake_news',\n",
              " 'IsaacBot/GP-Sentiment',\n",
              " 'Jack0508/test',\n",
              " 'Jikiwa/random_repo',\n",
              " 'Gabriel/quora_swe',\n",
              " 'Husain/intent-classification-en-fr',\n",
              " 'Intel/WEC-Eng',\n",
              " 'Ishwar/Senti',\n",
              " 'JIWON/nil_dataset',\n",
              " 'Jikiwa/demo2',\n",
              " 'Jikiwa/test-16336477963335',\n",
              " 'Jikiwa/test-16344362261113',\n",
              " 'KTH/speechdat',\n",
              " 'GEM/opusparcus',\n",
              " 'GroNLP/ik-nlp-22_winemag',\n",
              " 'HHousen/quora',\n",
              " 'Jikiwa/test-16336486877862',\n",
              " 'Jikiwa/test-16344347234752',\n",
              " 'JustinE/Test',\n",
              " 'GEM-submissions/GEM__bart_base_schema_guided_dialog__1645547915',\n",
              " 'HHousen/msrp',\n",
              " 'IFSTalfredoswald/MBTI',\n",
              " 'Jack0508/vi-ko-TED-txt',\n",
              " 'Jean-Baptiste/wikiner_fr',\n",
              " 'Jikiwa/demo1',\n",
              " 'KETI-AIR/klue',\n",
              " 'Khanoooo/autonlp-data-Corona',\n",
              " 'GEM/RotoWire_English-German',\n",
              " 'GEM/SciDuet',\n",
              " 'Jikiwa/pushedd-to-hub',\n",
              " 'Jikiwa/test-16340052901609',\n",
              " 'Jikiwa/test-16344364230608',\n",
              " 'KETI-AIR/nikl',\n",
              " 'GroNLP/ik-nlp-22_transqe',\n",
              " 'Helsinki-NLP/tatoeba_mt',\n",
              " 'IGESML/pubmed_neg',\n",
              " 'JesseParvess/book_snippets_asr',\n",
              " 'Jikiwa/pushed-to-hub',\n",
              " 'Jikiwa/test-16344360501144',\n",
              " 'JonathanSum/en_corpora_parliament_processed',\n",
              " 'Karavet/ILUR-news-text-classification-corpus',\n",
              " 'GEM/wiki_cat_sum',\n",
              " 'Hellisotherpeople/DebateSum',\n",
              " ...]"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset"
      ],
      "metadata": {
        "id": "19LuraDx1B-1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The second argument indicates languages, and the options include:\n",
        "# \"all_languages\", \"de\", \"en\", \"es\", \"fr\", \"ja\", \"zh\"\n",
        "dataset = load_dataset(\"amazon_reviews_multi\", \"en\")"
      ],
      "metadata": {
        "id": "Mlf_lG1x00tz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Amazon multilingual reviews"
      ],
      "metadata": {
        "id": "v9OQbohRyDWT"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### DatasetDict"
      ],
      "metadata": {
        "id": "aN4EoBrX8qM9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ccAgIlrm3yEX",
        "outputId": "eba649c3-5fe6-4632-84a2-43f6bba9291c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['review_id', 'product_id', 'reviewer_id', 'stars', 'review_body', 'review_title', 'language', 'product_category'],\n",
              "        num_rows: 200000\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['review_id', 'product_id', 'reviewer_id', 'stars', 'review_body', 'review_title', 'language', 'product_category'],\n",
              "        num_rows: 5000\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['review_id', 'product_id', 'reviewer_id', 'stars', 'review_body', 'review_title', 'language', 'product_category'],\n",
              "        num_rows: 5000\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def show_samples(dataset, num_samples=3, seed=42):\n",
        "    sample = dataset[\"train\"].shuffle(seed=seed).select(range(num_samples))\n",
        "    for example in sample:\n",
        "        print(f\"\\n'>> Title: {example['review_title']}'\")\n",
        "        print(f\"'>> Review: {example['review_body']}'\")\n",
        "        print(f\"'>> Product Category: {example['product_category']}'\")\n",
        "        print(f\"'>> Review stars: {example['stars']}'\")"
      ],
      "metadata": {
        "id": "T29Oc9LE5_ie"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "show_samples(dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dAbqEiTN6gg2",
        "outputId": "6e9d85a6-57fe-4022-eb88-a6a5fd1827b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "'>> Title: Worked in front position, not rear'\n",
            "'>> Review: 3 stars because these are not rear brakes as stated in the item description. At least the mount adapter only worked on the front fork of the bike that I got it for.'\n",
            "'>> Product Category: other'\n",
            "'>> Review stars: 3'\n",
            "\n",
            "'>> Title: meh'\n",
            "'>> Review: Does it’s job and it’s gorgeous but mine is falling apart, I had to basically put it together again with hot glue'\n",
            "'>> Product Category: beauty'\n",
            "'>> Review stars: 2'\n",
            "\n",
            "'>> Title: Can't beat these for the money'\n",
            "'>> Review: Bought this for handling miscellaneous aircraft parts and hanger \"stuff\" that I needed to organize; it really fit the bill. The unit arrived quickly, was well packaged and arrived intact (always a good sign). There are five wall mounts-- three on the top and two on the bottom. I wanted to mount it on the wall, so all I had to do was to remove the top two layers of plastic drawers, as well as the bottom corner drawers, place it when I wanted and mark it; I then used some of the new plastic screw in wall anchors (the 50 pound variety) and it easily mounted to the wall. Some have remarked that they wanted dividers for the drawers, and that they made those. Good idea. My application was that I needed something that I can see the contents at about eye level, so I wanted the fuller-sized drawers. I also like that these are the new plastic that doesn't get brittle and split like my older plastic drawers did. I like the all-plastic construction. It's heavy duty enough to hold metal parts, but being made of plastic it's not as heavy as a metal frame, so you can easily mount it to the wall and still load it up with heavy stuff, or light stuff. No problem there. For the money, you can't beat it. Best one of these I've bought to date-- and I've been using some version of these for over forty years.'\n",
            "'>> Product Category: other'\n",
            "'>> Review stars: 5'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Pandas DataFrame"
      ],
      "metadata": {
        "id": "ivXWwyIT8k1o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset.set_format(\"pandas\")\n",
        "df = dataset[\"test\"][:]\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "stfRCjPO65ph",
        "outputId": "17c8cec9-d36e-49cf-bf57-be7d63289a8f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       review_id          product_id          reviewer_id  stars  \\\n",
              "0     en_0199937  product_en_0902516  reviewer_en_0097389      1   \n",
              "1     en_0863335  product_en_0348072  reviewer_en_0601537      1   \n",
              "2     en_0565010  product_en_0356154  reviewer_en_0970351      1   \n",
              "3     en_0963290  product_en_0583322  reviewer_en_0216125      1   \n",
              "4     en_0238156  product_en_0487636  reviewer_en_0514203      1   \n",
              "...          ...                 ...                  ...    ...   \n",
              "4995  en_0681530  product_en_0198517  reviewer_en_0539725      5   \n",
              "4996  en_0157408  product_en_0206198  reviewer_en_0605408      5   \n",
              "4997  en_0542558  product_en_0525205  reviewer_en_0727937      5   \n",
              "4998  en_0754850  product_en_0613095  reviewer_en_0643466      5   \n",
              "4999  en_0843378  product_en_0323741  reviewer_en_0613446      5   \n",
              "\n",
              "                                            review_body  \\\n",
              "0     These are AWFUL. They are see through, the fab...   \n",
              "1     I bought 4 and NONE of them worked. Yes I used...   \n",
              "2     On first use it didn't heat up and now it does...   \n",
              "3     You want an HONEST answer? I just returned fro...   \n",
              "4     The glue works fine but the container is impos...   \n",
              "...                                                 ...   \n",
              "4995  It's still early, but so far the scales seems ...   \n",
              "4996  I’ve used this face mask a few times now and i...   \n",
              "4997                         Great price. Good quality.   \n",
              "4998  Fits my 2016 F150 XLT great. Now I dont have t...   \n",
              "4999  Just what I needed! Perfect for western theme ...   \n",
              "\n",
              "                                           review_title language  \\\n",
              "0                                Don’t waste your time!       en   \n",
              "1                                              One Star       en   \n",
              "2                                       Totally useless       en   \n",
              "3                                  Gold filled earrings       en   \n",
              "4                                        Poor container       en   \n",
              "...                                                 ...      ...   \n",
              "4995                                        Very sleek.       en   \n",
              "4996  ... face mask a few times now and it works gre...       en   \n",
              "4997                                         Five Stars       en   \n",
              "4998                                      2016 F150 XLT       en   \n",
              "4999                                           Perfect!       en   \n",
              "\n",
              "         product_category  \n",
              "0                 apparel  \n",
              "1                   other  \n",
              "2                   other  \n",
              "3                 jewelry  \n",
              "4     industrial_supplies  \n",
              "...                   ...  \n",
              "4995            drugstore  \n",
              "4996               beauty  \n",
              "4997                 home  \n",
              "4998           automotive  \n",
              "4999              apparel  \n",
              "\n",
              "[5000 rows x 8 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-75b17851-c3e6-415f-907e-786053276ab2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>reviewer_id</th>\n",
              "      <th>stars</th>\n",
              "      <th>review_body</th>\n",
              "      <th>review_title</th>\n",
              "      <th>language</th>\n",
              "      <th>product_category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>en_0199937</td>\n",
              "      <td>product_en_0902516</td>\n",
              "      <td>reviewer_en_0097389</td>\n",
              "      <td>1</td>\n",
              "      <td>These are AWFUL. They are see through, the fab...</td>\n",
              "      <td>Don’t waste your time!</td>\n",
              "      <td>en</td>\n",
              "      <td>apparel</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>en_0863335</td>\n",
              "      <td>product_en_0348072</td>\n",
              "      <td>reviewer_en_0601537</td>\n",
              "      <td>1</td>\n",
              "      <td>I bought 4 and NONE of them worked. Yes I used...</td>\n",
              "      <td>One Star</td>\n",
              "      <td>en</td>\n",
              "      <td>other</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>en_0565010</td>\n",
              "      <td>product_en_0356154</td>\n",
              "      <td>reviewer_en_0970351</td>\n",
              "      <td>1</td>\n",
              "      <td>On first use it didn't heat up and now it does...</td>\n",
              "      <td>Totally useless</td>\n",
              "      <td>en</td>\n",
              "      <td>other</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>en_0963290</td>\n",
              "      <td>product_en_0583322</td>\n",
              "      <td>reviewer_en_0216125</td>\n",
              "      <td>1</td>\n",
              "      <td>You want an HONEST answer? I just returned fro...</td>\n",
              "      <td>Gold filled earrings</td>\n",
              "      <td>en</td>\n",
              "      <td>jewelry</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>en_0238156</td>\n",
              "      <td>product_en_0487636</td>\n",
              "      <td>reviewer_en_0514203</td>\n",
              "      <td>1</td>\n",
              "      <td>The glue works fine but the container is impos...</td>\n",
              "      <td>Poor container</td>\n",
              "      <td>en</td>\n",
              "      <td>industrial_supplies</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4995</th>\n",
              "      <td>en_0681530</td>\n",
              "      <td>product_en_0198517</td>\n",
              "      <td>reviewer_en_0539725</td>\n",
              "      <td>5</td>\n",
              "      <td>It's still early, but so far the scales seems ...</td>\n",
              "      <td>Very sleek.</td>\n",
              "      <td>en</td>\n",
              "      <td>drugstore</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4996</th>\n",
              "      <td>en_0157408</td>\n",
              "      <td>product_en_0206198</td>\n",
              "      <td>reviewer_en_0605408</td>\n",
              "      <td>5</td>\n",
              "      <td>I’ve used this face mask a few times now and i...</td>\n",
              "      <td>... face mask a few times now and it works gre...</td>\n",
              "      <td>en</td>\n",
              "      <td>beauty</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4997</th>\n",
              "      <td>en_0542558</td>\n",
              "      <td>product_en_0525205</td>\n",
              "      <td>reviewer_en_0727937</td>\n",
              "      <td>5</td>\n",
              "      <td>Great price. Good quality.</td>\n",
              "      <td>Five Stars</td>\n",
              "      <td>en</td>\n",
              "      <td>home</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4998</th>\n",
              "      <td>en_0754850</td>\n",
              "      <td>product_en_0613095</td>\n",
              "      <td>reviewer_en_0643466</td>\n",
              "      <td>5</td>\n",
              "      <td>Fits my 2016 F150 XLT great. Now I dont have t...</td>\n",
              "      <td>2016 F150 XLT</td>\n",
              "      <td>en</td>\n",
              "      <td>automotive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4999</th>\n",
              "      <td>en_0843378</td>\n",
              "      <td>product_en_0323741</td>\n",
              "      <td>reviewer_en_0613446</td>\n",
              "      <td>5</td>\n",
              "      <td>Just what I needed! Perfect for western theme ...</td>\n",
              "      <td>Perfect!</td>\n",
              "      <td>en</td>\n",
              "      <td>apparel</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5000 rows × 8 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-75b17851-c3e6-415f-907e-786053276ab2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-75b17851-c3e6-415f-907e-786053276ab2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-75b17851-c3e6-415f-907e-786053276ab2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Miscellaneous "
      ],
      "metadata": {
        "id": "bOqyHi3IfIBb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## TMDB movie plots"
      ],
      "metadata": {
        "id": "eQpJeUbLfQft"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "TGAS3yBjfeAn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "raw_df = pd.read_csv('https://github.com/dipanjanS/nlp_workshop_dhs18/raw/master/Unit%2010%20-%20Project%208%20-%20Movie%20Recommendations%20with%20Document%20Similarity/tmdb_5000_movies.csv.gz', compression='gzip')\n",
        "raw_df.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x-9fgt2CfVP8",
        "outputId": "81ba5390-054d-4b09-e0d6-688c5701003a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 4803 entries, 0 to 4802\n",
            "Data columns (total 20 columns):\n",
            " #   Column                Non-Null Count  Dtype  \n",
            "---  ------                --------------  -----  \n",
            " 0   budget                4803 non-null   int64  \n",
            " 1   genres                4803 non-null   object \n",
            " 2   homepage              1712 non-null   object \n",
            " 3   id                    4803 non-null   int64  \n",
            " 4   keywords              4803 non-null   object \n",
            " 5   original_language     4803 non-null   object \n",
            " 6   original_title        4803 non-null   object \n",
            " 7   overview              4800 non-null   object \n",
            " 8   popularity            4803 non-null   float64\n",
            " 9   production_companies  4803 non-null   object \n",
            " 10  production_countries  4803 non-null   object \n",
            " 11  release_date          4802 non-null   object \n",
            " 12  revenue               4803 non-null   int64  \n",
            " 13  runtime               4801 non-null   float64\n",
            " 14  spoken_languages      4803 non-null   object \n",
            " 15  status                4803 non-null   object \n",
            " 16  tagline               3959 non-null   object \n",
            " 17  title                 4803 non-null   object \n",
            " 18  vote_average          4803 non-null   float64\n",
            " 19  vote_count            4803 non-null   int64  \n",
            "dtypes: float64(3), int64(4), object(13)\n",
            "memory usage: 750.6+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 739
        },
        "id": "bX01N2QigIRa",
        "outputId": "0d48f51a-9f08-4730-a361-94a7bc444a65"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      budget                                             genres  \\\n",
              "0  237000000  [{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam...   \n",
              "1  300000000  [{\"id\": 12, \"name\": \"Adventure\"}, {\"id\": 14, \"...   \n",
              "2  245000000  [{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam...   \n",
              "3  250000000  [{\"id\": 28, \"name\": \"Action\"}, {\"id\": 80, \"nam...   \n",
              "4  260000000  [{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam...   \n",
              "\n",
              "                                       homepage      id  \\\n",
              "0                   http://www.avatarmovie.com/   19995   \n",
              "1  http://disney.go.com/disneypictures/pirates/     285   \n",
              "2   http://www.sonypictures.com/movies/spectre/  206647   \n",
              "3            http://www.thedarkknightrises.com/   49026   \n",
              "4          http://movies.disney.com/john-carter   49529   \n",
              "\n",
              "                                            keywords original_language  \\\n",
              "0  [{\"id\": 1463, \"name\": \"culture clash\"}, {\"id\":...                en   \n",
              "1  [{\"id\": 270, \"name\": \"ocean\"}, {\"id\": 726, \"na...                en   \n",
              "2  [{\"id\": 470, \"name\": \"spy\"}, {\"id\": 818, \"name...                en   \n",
              "3  [{\"id\": 849, \"name\": \"dc comics\"}, {\"id\": 853,...                en   \n",
              "4  [{\"id\": 818, \"name\": \"based on novel\"}, {\"id\":...                en   \n",
              "\n",
              "                             original_title  \\\n",
              "0                                    Avatar   \n",
              "1  Pirates of the Caribbean: At World's End   \n",
              "2                                   Spectre   \n",
              "3                     The Dark Knight Rises   \n",
              "4                               John Carter   \n",
              "\n",
              "                                            overview  popularity  \\\n",
              "0  In the 22nd century, a paraplegic Marine is di...  150.437577   \n",
              "1  Captain Barbossa, long believed to be dead, ha...  139.082615   \n",
              "2  A cryptic message from Bond’s past sends him o...  107.376788   \n",
              "3  Following the death of District Attorney Harve...  112.312950   \n",
              "4  John Carter is a war-weary, former military ca...   43.926995   \n",
              "\n",
              "                                production_companies  \\\n",
              "0  [{\"name\": \"Ingenious Film Partners\", \"id\": 289...   \n",
              "1  [{\"name\": \"Walt Disney Pictures\", \"id\": 2}, {\"...   \n",
              "2  [{\"name\": \"Columbia Pictures\", \"id\": 5}, {\"nam...   \n",
              "3  [{\"name\": \"Legendary Pictures\", \"id\": 923}, {\"...   \n",
              "4        [{\"name\": \"Walt Disney Pictures\", \"id\": 2}]   \n",
              "\n",
              "                                production_countries release_date     revenue  \\\n",
              "0  [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   2009-12-10  2787965087   \n",
              "1  [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   2007-05-19   961000000   \n",
              "2  [{\"iso_3166_1\": \"GB\", \"name\": \"United Kingdom\"...   2015-10-26   880674609   \n",
              "3  [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   2012-07-16  1084939099   \n",
              "4  [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   2012-03-07   284139100   \n",
              "\n",
              "   runtime                                   spoken_languages    status  \\\n",
              "0    162.0  [{\"iso_639_1\": \"en\", \"name\": \"English\"}, {\"iso...  Released   \n",
              "1    169.0           [{\"iso_639_1\": \"en\", \"name\": \"English\"}]  Released   \n",
              "2    148.0  [{\"iso_639_1\": \"fr\", \"name\": \"Fran\\u00e7ais\"},...  Released   \n",
              "3    165.0           [{\"iso_639_1\": \"en\", \"name\": \"English\"}]  Released   \n",
              "4    132.0           [{\"iso_639_1\": \"en\", \"name\": \"English\"}]  Released   \n",
              "\n",
              "                                          tagline  \\\n",
              "0                     Enter the World of Pandora.   \n",
              "1  At the end of the world, the adventure begins.   \n",
              "2                           A Plan No One Escapes   \n",
              "3                                 The Legend Ends   \n",
              "4            Lost in our world, found in another.   \n",
              "\n",
              "                                      title  vote_average  vote_count  \n",
              "0                                    Avatar           7.2       11800  \n",
              "1  Pirates of the Caribbean: At World's End           6.9        4500  \n",
              "2                                   Spectre           6.3        4466  \n",
              "3                     The Dark Knight Rises           7.6        9106  \n",
              "4                               John Carter           6.1        2124  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-85852e21-3029-404f-8073-a0b7b2238f4c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>budget</th>\n",
              "      <th>genres</th>\n",
              "      <th>homepage</th>\n",
              "      <th>id</th>\n",
              "      <th>keywords</th>\n",
              "      <th>original_language</th>\n",
              "      <th>original_title</th>\n",
              "      <th>overview</th>\n",
              "      <th>popularity</th>\n",
              "      <th>production_companies</th>\n",
              "      <th>production_countries</th>\n",
              "      <th>release_date</th>\n",
              "      <th>revenue</th>\n",
              "      <th>runtime</th>\n",
              "      <th>spoken_languages</th>\n",
              "      <th>status</th>\n",
              "      <th>tagline</th>\n",
              "      <th>title</th>\n",
              "      <th>vote_average</th>\n",
              "      <th>vote_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>237000000</td>\n",
              "      <td>[{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam...</td>\n",
              "      <td>http://www.avatarmovie.com/</td>\n",
              "      <td>19995</td>\n",
              "      <td>[{\"id\": 1463, \"name\": \"culture clash\"}, {\"id\":...</td>\n",
              "      <td>en</td>\n",
              "      <td>Avatar</td>\n",
              "      <td>In the 22nd century, a paraplegic Marine is di...</td>\n",
              "      <td>150.437577</td>\n",
              "      <td>[{\"name\": \"Ingenious Film Partners\", \"id\": 289...</td>\n",
              "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
              "      <td>2009-12-10</td>\n",
              "      <td>2787965087</td>\n",
              "      <td>162.0</td>\n",
              "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}, {\"iso...</td>\n",
              "      <td>Released</td>\n",
              "      <td>Enter the World of Pandora.</td>\n",
              "      <td>Avatar</td>\n",
              "      <td>7.2</td>\n",
              "      <td>11800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>300000000</td>\n",
              "      <td>[{\"id\": 12, \"name\": \"Adventure\"}, {\"id\": 14, \"...</td>\n",
              "      <td>http://disney.go.com/disneypictures/pirates/</td>\n",
              "      <td>285</td>\n",
              "      <td>[{\"id\": 270, \"name\": \"ocean\"}, {\"id\": 726, \"na...</td>\n",
              "      <td>en</td>\n",
              "      <td>Pirates of the Caribbean: At World's End</td>\n",
              "      <td>Captain Barbossa, long believed to be dead, ha...</td>\n",
              "      <td>139.082615</td>\n",
              "      <td>[{\"name\": \"Walt Disney Pictures\", \"id\": 2}, {\"...</td>\n",
              "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
              "      <td>2007-05-19</td>\n",
              "      <td>961000000</td>\n",
              "      <td>169.0</td>\n",
              "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}]</td>\n",
              "      <td>Released</td>\n",
              "      <td>At the end of the world, the adventure begins.</td>\n",
              "      <td>Pirates of the Caribbean: At World's End</td>\n",
              "      <td>6.9</td>\n",
              "      <td>4500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>245000000</td>\n",
              "      <td>[{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam...</td>\n",
              "      <td>http://www.sonypictures.com/movies/spectre/</td>\n",
              "      <td>206647</td>\n",
              "      <td>[{\"id\": 470, \"name\": \"spy\"}, {\"id\": 818, \"name...</td>\n",
              "      <td>en</td>\n",
              "      <td>Spectre</td>\n",
              "      <td>A cryptic message from Bond’s past sends him o...</td>\n",
              "      <td>107.376788</td>\n",
              "      <td>[{\"name\": \"Columbia Pictures\", \"id\": 5}, {\"nam...</td>\n",
              "      <td>[{\"iso_3166_1\": \"GB\", \"name\": \"United Kingdom\"...</td>\n",
              "      <td>2015-10-26</td>\n",
              "      <td>880674609</td>\n",
              "      <td>148.0</td>\n",
              "      <td>[{\"iso_639_1\": \"fr\", \"name\": \"Fran\\u00e7ais\"},...</td>\n",
              "      <td>Released</td>\n",
              "      <td>A Plan No One Escapes</td>\n",
              "      <td>Spectre</td>\n",
              "      <td>6.3</td>\n",
              "      <td>4466</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>250000000</td>\n",
              "      <td>[{\"id\": 28, \"name\": \"Action\"}, {\"id\": 80, \"nam...</td>\n",
              "      <td>http://www.thedarkknightrises.com/</td>\n",
              "      <td>49026</td>\n",
              "      <td>[{\"id\": 849, \"name\": \"dc comics\"}, {\"id\": 853,...</td>\n",
              "      <td>en</td>\n",
              "      <td>The Dark Knight Rises</td>\n",
              "      <td>Following the death of District Attorney Harve...</td>\n",
              "      <td>112.312950</td>\n",
              "      <td>[{\"name\": \"Legendary Pictures\", \"id\": 923}, {\"...</td>\n",
              "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
              "      <td>2012-07-16</td>\n",
              "      <td>1084939099</td>\n",
              "      <td>165.0</td>\n",
              "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}]</td>\n",
              "      <td>Released</td>\n",
              "      <td>The Legend Ends</td>\n",
              "      <td>The Dark Knight Rises</td>\n",
              "      <td>7.6</td>\n",
              "      <td>9106</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>260000000</td>\n",
              "      <td>[{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam...</td>\n",
              "      <td>http://movies.disney.com/john-carter</td>\n",
              "      <td>49529</td>\n",
              "      <td>[{\"id\": 818, \"name\": \"based on novel\"}, {\"id\":...</td>\n",
              "      <td>en</td>\n",
              "      <td>John Carter</td>\n",
              "      <td>John Carter is a war-weary, former military ca...</td>\n",
              "      <td>43.926995</td>\n",
              "      <td>[{\"name\": \"Walt Disney Pictures\", \"id\": 2}]</td>\n",
              "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
              "      <td>2012-03-07</td>\n",
              "      <td>284139100</td>\n",
              "      <td>132.0</td>\n",
              "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}]</td>\n",
              "      <td>Released</td>\n",
              "      <td>Lost in our world, found in another.</td>\n",
              "      <td>John Carter</td>\n",
              "      <td>6.1</td>\n",
              "      <td>2124</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-85852e21-3029-404f-8073-a0b7b2238f4c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-85852e21-3029-404f-8073-a0b7b2238f4c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-85852e21-3029-404f-8073-a0b7b2238f4c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def load_tmdb():\n",
        "    raw_df = pd.read_csv('https://github.com/dipanjanS/nlp_workshop_dhs18/raw/master/Unit%2010%20-%20Project%208%20-%20Movie%20Recommendations%20with%20Document%20Similarity/tmdb_5000_movies.csv.gz', compression='gzip')\n",
        "    cols = ['title', 'tagline', 'overview', 'popularity', 'release_date']\n",
        "    df = raw_df[cols].copy()\n",
        "    df['tagline'].fillna('', inplace=True)\n",
        "    df['description'] = df['tagline'] + ' ' + df['overview']\n",
        "    df.set_index('release_date', inplace=True)\n",
        "    df.sort_index(inplace=True)\n",
        "    return df"
      ],
      "metadata": {
        "id": "E4a729ykiKpe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tmdb = load_tmdb()\n",
        "tmdb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "3JgeQH2kjuk-",
        "outputId": "58619aa9-9ba4-4b02-fa8f-2816f5e6988e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                   title  \\\n",
              "release_date                               \n",
              "1916-09-04                   Intolerance   \n",
              "1925-11-05                The Big Parade   \n",
              "1927-01-10                    Metropolis   \n",
              "1929-01-30                 Pandora's Box   \n",
              "1929-02-08           The Broadway Melody   \n",
              "...                                  ...   \n",
              "2016-09-09         The Birth of a Nation   \n",
              "2016-09-16                    Mr. Church   \n",
              "2016-10-02         Two Lovers and a Bear   \n",
              "2017-02-03              Growing Up Smith   \n",
              "NaN           America Is Still the Place   \n",
              "\n",
              "                                                        tagline  \\\n",
              "release_date                                                      \n",
              "1916-09-04                        The Cruel Hand of Intolerance   \n",
              "1925-11-05                                                        \n",
              "1927-01-10    There can be no understanding between the hand...   \n",
              "1929-01-30                                                        \n",
              "1929-02-08    The pulsating drama of Broadway's bared heart ...   \n",
              "...                                                         ...   \n",
              "2016-09-09                       The Untold Story of Nat Turner   \n",
              "2016-09-16     He was the one person she could always count on.   \n",
              "2016-10-02                                                        \n",
              "2017-02-03             It’s better to stand out than to fit in.   \n",
              "NaN                                                               \n",
              "\n",
              "                                                       overview  popularity  \\\n",
              "release_date                                                                  \n",
              "1916-09-04    The story of a poor young woman, separated by ...    3.232447   \n",
              "1925-11-05    The story of an idle rich boy who joins the US...    0.785744   \n",
              "1927-01-10    In a futuristic city sharply divided between t...   32.351527   \n",
              "1929-01-30    The rise and inevitable fall of an amoral but ...    1.824184   \n",
              "1929-02-08    Harriet and Queenie Mahoney, a vaudeville act,...    0.968865   \n",
              "...                                                         ...         ...   \n",
              "2016-09-09    Nat Turner, a former slave in America, leads a...    9.452808   \n",
              "2016-09-16    A unique friendship develops when a little gir...    7.828459   \n",
              "2016-10-02    Set in a small town near the North Pole where ...    3.139294   \n",
              "2017-02-03    In 1979, an Indian family moves to America wit...    0.710870   \n",
              "NaN           1971 post civil rights San Francisco seemed li...    0.000000   \n",
              "\n",
              "                                                    description  \n",
              "release_date                                                     \n",
              "1916-09-04    The Cruel Hand of Intolerance The story of a p...  \n",
              "1925-11-05     The story of an idle rich boy who joins the U...  \n",
              "1927-01-10    There can be no understanding between the hand...  \n",
              "1929-01-30     The rise and inevitable fall of an amoral but...  \n",
              "1929-02-08    The pulsating drama of Broadway's bared heart ...  \n",
              "...                                                         ...  \n",
              "2016-09-09    The Untold Story of Nat Turner Nat Turner, a f...  \n",
              "2016-09-16    He was the one person she could always count o...  \n",
              "2016-10-02     Set in a small town near the North Pole where...  \n",
              "2017-02-03    It’s better to stand out than to fit in. In 19...  \n",
              "NaN            1971 post civil rights San Francisco seemed l...  \n",
              "\n",
              "[4803 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2de8fb8e-b577-4517-a7e6-91d8fc53a4ce\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>tagline</th>\n",
              "      <th>overview</th>\n",
              "      <th>popularity</th>\n",
              "      <th>description</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>release_date</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1916-09-04</th>\n",
              "      <td>Intolerance</td>\n",
              "      <td>The Cruel Hand of Intolerance</td>\n",
              "      <td>The story of a poor young woman, separated by ...</td>\n",
              "      <td>3.232447</td>\n",
              "      <td>The Cruel Hand of Intolerance The story of a p...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1925-11-05</th>\n",
              "      <td>The Big Parade</td>\n",
              "      <td></td>\n",
              "      <td>The story of an idle rich boy who joins the US...</td>\n",
              "      <td>0.785744</td>\n",
              "      <td>The story of an idle rich boy who joins the U...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1927-01-10</th>\n",
              "      <td>Metropolis</td>\n",
              "      <td>There can be no understanding between the hand...</td>\n",
              "      <td>In a futuristic city sharply divided between t...</td>\n",
              "      <td>32.351527</td>\n",
              "      <td>There can be no understanding between the hand...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1929-01-30</th>\n",
              "      <td>Pandora's Box</td>\n",
              "      <td></td>\n",
              "      <td>The rise and inevitable fall of an amoral but ...</td>\n",
              "      <td>1.824184</td>\n",
              "      <td>The rise and inevitable fall of an amoral but...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1929-02-08</th>\n",
              "      <td>The Broadway Melody</td>\n",
              "      <td>The pulsating drama of Broadway's bared heart ...</td>\n",
              "      <td>Harriet and Queenie Mahoney, a vaudeville act,...</td>\n",
              "      <td>0.968865</td>\n",
              "      <td>The pulsating drama of Broadway's bared heart ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2016-09-09</th>\n",
              "      <td>The Birth of a Nation</td>\n",
              "      <td>The Untold Story of Nat Turner</td>\n",
              "      <td>Nat Turner, a former slave in America, leads a...</td>\n",
              "      <td>9.452808</td>\n",
              "      <td>The Untold Story of Nat Turner Nat Turner, a f...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2016-09-16</th>\n",
              "      <td>Mr. Church</td>\n",
              "      <td>He was the one person she could always count on.</td>\n",
              "      <td>A unique friendship develops when a little gir...</td>\n",
              "      <td>7.828459</td>\n",
              "      <td>He was the one person she could always count o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2016-10-02</th>\n",
              "      <td>Two Lovers and a Bear</td>\n",
              "      <td></td>\n",
              "      <td>Set in a small town near the North Pole where ...</td>\n",
              "      <td>3.139294</td>\n",
              "      <td>Set in a small town near the North Pole where...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2017-02-03</th>\n",
              "      <td>Growing Up Smith</td>\n",
              "      <td>It’s better to stand out than to fit in.</td>\n",
              "      <td>In 1979, an Indian family moves to America wit...</td>\n",
              "      <td>0.710870</td>\n",
              "      <td>It’s better to stand out than to fit in. In 19...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NaN</th>\n",
              "      <td>America Is Still the Place</td>\n",
              "      <td></td>\n",
              "      <td>1971 post civil rights San Francisco seemed li...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1971 post civil rights San Francisco seemed l...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4803 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2de8fb8e-b577-4517-a7e6-91d8fc53a4ce')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2de8fb8e-b577-4517-a7e6-91d8fc53a4ce button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2de8fb8e-b577-4517-a7e6-91d8fc53a4ce');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    }
  ]
}