Cleaning-1.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "rxEtp44DO1_K"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "pI2xk9HBPFJc"
      },
      "outputs": [],
      "source": [
        "dataset_path = os.getcwd() + '\\Datasets\\Original\\\\'\n",
        "\n",
        "test_full = pd.read_csv(dataset_path + 'full_test_plaintext.txt', sep = '\\t')\n",
        "train_full = pd.read_csv(dataset_path + 'full_train_plaintext.txt', sep = '\\t')\n",
        "valid_full = pd.read_csv(dataset_path + 'full_valid_plaintext.txt', sep = '\\t')\n",
        "\n",
        "test_img = pd.read_csv(dataset_path + 'img_test_plaintext.txt', sep = '\\t')\n",
        "train_img = pd.read_csv(dataset_path + 'img_train_plaintext.txt', sep = '\\t')\n",
        "valid_img = pd.read_csv(dataset_path + 'img_valid_plaintext.txt', sep = '\\t')\n",
        "\n",
        "test_balanced = pd.read_csv(dataset_path + 'balanced_test_plaintext.txt', sep = '\\t')\n",
        "\n",
        "emoji_map = pd.read_csv(dataset_path + 'emoji_map_1791.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rUvDrssJXf0b"
      },
      "source": [
        "# Sanity Check"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-HfK1gngoWE2"
      },
      "source": [
        "**Key Points:**\n",
        "\n",
        "1. Tweets that contain a quoted tweet ends with _q (train, test, valid_img do not contain _q)\n",
        "\n",
        "2. For tweets that contain a quoted tweet with pic, that tweet ID is not included in img.txt, but the text in the quoted tweet is considered. (See codes below)\n",
        "e.g. id 742700829703057408_q is in full_test but not img_test\n",
        "\n",
        "3. Idk for this case, 742886817339432962_q.. annotations: 1 emoji that belongs to the quoted tweet. However, we have two other emojis in the tweet itself. 747532397315395584_q, 743594076503683072_q, 747522541464453120_q, 744959564504793088_q too\n",
        "\n",
        "4. 747159552718680064_q have 4 emojis in main tweet but only 1 annotation..\n",
        "Conclusion: I think they only see the quoted tweet not the main tweet if that tweet contains a quoted tweet\n",
        "\n",
        "5. 753324941404147712 2 annotations: flower and heart -> heart from the pic??, but 742960387843338241 doesnt take into account the emoji in the pic\n",
        "\n",
        "6. 746283452740206592 weird annotations. In full_train_plaintext only hv 5 annotations, img_test 4 annotations, img_train -> a lot\n",
        "\n",
        "It's a reply to a main tweet and all the replies in that thread are considered.\n",
        "Try this \"emoji_map.loc[[768, 775, 520, 776, 525, 784, 785, 787, 1556, 791, 537, 1562, 539, 540, 796, 1310, 798, 544, 806, 554, 557, 1582, 815, 818, 1330, 566, 567, 56, 569, 58, 57, 824, 1598, 577, 1604, 585, 1354, 593, 1361, 1362, 1365, 1367, 600, 1119, 1125, 1126, 615, 103, 1389, 1134, 1144, 1406, 130, 642, 648, 649, 1162, 651, 652, 1167, 143, 1169, 913, 916, 662, 663, 664, 919, 920, 1180, 1437, 670, 926, 930, 681, 684, 174, 1200, 1201, 177, 1210, 1217, 200, 721, 1255, 1263, 754, 757, 509, 510], 'ucode']\"\n",
        "\n",
        "Maybe the API will answer every doubt we have :)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "d9hE295pWgGG",
        "outputId": "de17b446-ac35-4d64-8c39-1ba8771dc881"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total test rows: 1000000\n",
            "Total test tweets containing a quoted tweet: 53482\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>747643690521341953</td>\n",
              "      <td>1138</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>746196870121164800</td>\n",
              "      <td>1056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744595559470800896</td>\n",
              "      <td>1381</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>747720844730859520</td>\n",
              "      <td>889,1128</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>743285629576843264</td>\n",
              "      <td>1424</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   id annotations\n",
              "0  747643690521341953        1138\n",
              "1  746196870121164800        1056\n",
              "2  744595559470800896        1381\n",
              "3  747720844730859520    889,1128\n",
              "4  743285629576843264        1424"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Total test rows: \" + str(len(test_full)))\n",
        "print(\"Total test tweets containing a quoted tweet: \" + str(len(test_full[(test_full['id'].str[-2:] == \"_q\")])))\n",
        "test_full.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "_YGe6FanW-VM",
        "outputId": "5d31c22d-406d-4fc0-f3f1-6c1efa659af9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total training rows: 12997219\n",
            "Total training tweets containing a quoted tweet: 690524\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>744014442837454848</td>\n",
              "      <td>47,883</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>742407819496919041</td>\n",
              "      <td>1381</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744101567981359105</td>\n",
              "      <td>1421</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>744692442033577984</td>\n",
              "      <td>1056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>746426733834944512</td>\n",
              "      <td>1056,1381</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   id annotations\n",
              "0  744014442837454848      47,883\n",
              "1  742407819496919041        1381\n",
              "2  744101567981359105        1421\n",
              "3  744692442033577984        1056\n",
              "4  746426733834944512   1056,1381"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Total training rows: \" + str(len(train_full)))\n",
        "print(\"Total training tweets containing a quoted tweet: \" + str(len(train_full[(train_full['id'].str[-2:] == \"_q\")])))\n",
        "train_full.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "R_hTio4MW_9k",
        "outputId": "a1a5b60b-91db-414c-eb19-19e9cd0dee8f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total validation rows: 1000000\n",
            "Total validation tweets containing a quoted tweet: 53332\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>746634523048292352</td>\n",
              "      <td>1103,1389,1399</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>741650544847147009</td>\n",
              "      <td>1381</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>746803092239196162</td>\n",
              "      <td>1424</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>746625375875170304</td>\n",
              "      <td>1107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>744724790485757952</td>\n",
              "      <td>1620</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   id     annotations\n",
              "0  746634523048292352  1103,1389,1399\n",
              "1  741650544847147009            1381\n",
              "2  746803092239196162            1424\n",
              "3  746625375875170304            1107\n",
              "4  744724790485757952            1620"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Total validation rows: \" + str(len(valid_full)))\n",
        "print(\"Total validation tweets containing a quoted tweet: \" + str(len(valid_full[(valid_full['id'].str[-2:] == \"_q\")])))\n",
        "valid_full.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "QE1mfSmgXETj",
        "outputId": "27592065-dba9-4a42-e551-9561bf3af218"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total test image rows: 79191\n",
            "total test image tweets containing a quoted tweet: 0\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>imgid</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>748230468458418177</td>\n",
              "      <td>http://pbs.twimg.com/media/CmI_nUeWkAAoDLm.jpg</td>\n",
              "      <td>1382</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>747854910612398081</td>\n",
              "      <td>http://pbs.twimg.com/media/CmDqE7qUkAA4oZH.jpg</td>\n",
              "      <td>174,1108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>742432606709485568</td>\n",
              "      <td>http://pbs.twimg.com/media/Ci6Hi8xVEAAUkST.jpg</td>\n",
              "      <td>186</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>742257533616816128</td>\n",
              "      <td>http://pbs.twimg.com/media/Ckxuh4SXIAEQZQX.jpg</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>747312758127890432</td>\n",
              "      <td>http://pbs.twimg.com/media/Cl78_KTUsAAj-wC.jpg</td>\n",
              "      <td>156,601,1109</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   id                                           imgid  \\\n",
              "0  748230468458418177  http://pbs.twimg.com/media/CmI_nUeWkAAoDLm.jpg   \n",
              "1  747854910612398081  http://pbs.twimg.com/media/CmDqE7qUkAA4oZH.jpg   \n",
              "2  742432606709485568  http://pbs.twimg.com/media/Ci6Hi8xVEAAUkST.jpg   \n",
              "3  742257533616816128  http://pbs.twimg.com/media/Ckxuh4SXIAEQZQX.jpg   \n",
              "4  747312758127890432  http://pbs.twimg.com/media/Cl78_KTUsAAj-wC.jpg   \n",
              "\n",
              "    annotations  \n",
              "0          1382  \n",
              "1      174,1108  \n",
              "2           186  \n",
              "3            78  \n",
              "4  156,601,1109  "
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_img = test_img.astype({'id': str})\n",
        "print(\"Total test image rows: \" + str(len(test_img)))\n",
        "print(\"total test image tweets containing a quoted tweet: \" + str(len(test_img[(test_img['id'].str[-2:] == \"_q\")])))\n",
        "test_img.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "u38bggoWXIUb",
        "outputId": "551315ae-7c46-4140-c34e-d0d9f1573dd9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total training image rows: 916864\n",
            "Total training image tweets that containing a quoted tweet: 0\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>imgid</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>744014442837454848</td>\n",
              "      <td>http://pbs.twimg.com/media/ClNFMNSWgAI7qq0.jpg</td>\n",
              "      <td>47,883</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>742240402288345088</td>\n",
              "      <td>http://pbs.twimg.com/media/CkzuE33WsAA2cUB.jpg</td>\n",
              "      <td>229</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>742427564535353349</td>\n",
              "      <td>http://pbs.twimg.com/media/Ck2h7lzXIAA0Aqw.jpg</td>\n",
              "      <td>658</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>747888709375229952</td>\n",
              "      <td>http://pbs.twimg.com/media/CmEIe-_VYAADvg0.jpg</td>\n",
              "      <td>1108,1108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>747199027649871872</td>\n",
              "      <td>http://pbs.twimg.com/media/Cl6UJF1XIAADDAJ.jpg</td>\n",
              "      <td>1466</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   id                                           imgid  \\\n",
              "0  744014442837454848  http://pbs.twimg.com/media/ClNFMNSWgAI7qq0.jpg   \n",
              "1  742240402288345088  http://pbs.twimg.com/media/CkzuE33WsAA2cUB.jpg   \n",
              "2  742427564535353349  http://pbs.twimg.com/media/Ck2h7lzXIAA0Aqw.jpg   \n",
              "3  747888709375229952  http://pbs.twimg.com/media/CmEIe-_VYAADvg0.jpg   \n",
              "4  747199027649871872  http://pbs.twimg.com/media/Cl6UJF1XIAADDAJ.jpg   \n",
              "\n",
              "  annotations  \n",
              "0      47,883  \n",
              "1         229  \n",
              "2         658  \n",
              "3   1108,1108  \n",
              "4        1466  "
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_img = train_img.astype({'id': str})\n",
        "print(\"Total training image rows: \" + str(len(train_img)))\n",
        "print(\"Total training image tweets that containing a quoted tweet: \" + str(len(train_img[(train_img['id'].str[-2:] == \"_q\")])))\n",
        "train_img.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "OMhlGRGlXKqs",
        "outputId": "0cc5a9ac-1c71-47d4-c58b-94cca6579362"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total validation image rows: 79419\n",
            "total validation image tweets containing a quoted tweet: 0\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>imgid</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>746473363992784897</td>\n",
              "      <td>http://pbs.twimg.com/media/Clv7F6mVAAIw28L.jpg</td>\n",
              "      <td>1105</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>747463019911843840</td>\n",
              "      <td>http://pbs.twimg.com/media/Cl9sTJQWYAEm4q9.jpg</td>\n",
              "      <td>186,503,1111,1112,1113,1114</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>742355654052978689</td>\n",
              "      <td>http://pbs.twimg.com/media/Ck1ggjFUoAE58nB.jpg</td>\n",
              "      <td>186,635</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>748291920175173632</td>\n",
              "      <td>http://pbs.twimg.com/media/Cl7U_BbUsAEasQD.jpg</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>746296637509668866</td>\n",
              "      <td>http://pbs.twimg.com/media/Clrhp96UYAAQOmv.jpg</td>\n",
              "      <td>186,426</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   id                                           imgid  \\\n",
              "0  746473363992784897  http://pbs.twimg.com/media/Clv7F6mVAAIw28L.jpg   \n",
              "1  747463019911843840  http://pbs.twimg.com/media/Cl9sTJQWYAEm4q9.jpg   \n",
              "2  742355654052978689  http://pbs.twimg.com/media/Ck1ggjFUoAE58nB.jpg   \n",
              "3  748291920175173632  http://pbs.twimg.com/media/Cl7U_BbUsAEasQD.jpg   \n",
              "4  746296637509668866  http://pbs.twimg.com/media/Clrhp96UYAAQOmv.jpg   \n",
              "\n",
              "                   annotations  \n",
              "0                         1105  \n",
              "1  186,503,1111,1112,1113,1114  \n",
              "2                      186,635  \n",
              "3                           78  \n",
              "4                      186,426  "
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "valid_img = valid_img.astype({'id': str})\n",
        "print(\"Total validation image rows: \" + str(len(valid_img)))\n",
        "print(\"total validation image tweets containing a quoted tweet: \" + str(len(valid_img[(valid_img['id'].str[-2:] == \"_q\")])))\n",
        "valid_img.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "QaDPhmypXM07",
        "outputId": "d155cac3-8259-46be-cd50-6fd8bd46fa6b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total balanced rows: 9528\n",
            "total balanced tweets containing a quoted tweet: 659\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>742207272537886721</td>\n",
              "      <td>575</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>746284085362966528</td>\n",
              "      <td>24,25,186,187,217,550,560,979,980,1126,1219,12...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>741070803807043585_q</td>\n",
              "      <td>1516</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>744681934626791428</td>\n",
              "      <td>761,1088</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>747676925183803392</td>\n",
              "      <td>659</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                     id                                        annotations\n",
              "0    742207272537886721                                                575\n",
              "1    746284085362966528  24,25,186,187,217,550,560,979,980,1126,1219,12...\n",
              "2  741070803807043585_q                                               1516\n",
              "3    744681934626791428                                           761,1088\n",
              "4    747676925183803392                                                659"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Total balanced rows: \" + str(len(test_balanced)))\n",
        "print(\"total balanced tweets containing a quoted tweet: \" + str(len(test_balanced[(test_balanced['id'].str[-2:] == \"_q\")])))\n",
        "test_balanced.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "F4PfDipTXOey",
        "outputId": "61f08345-b4c5-4971-b162-2d9c4e92b3cf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Size of emoji map: (1791, 11)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>category</th>\n",
              "      <th>title</th>\n",
              "      <th>shorts</th>\n",
              "      <th>unicode_alternates</th>\n",
              "      <th>keywords</th>\n",
              "      <th>ucode_short</th>\n",
              "      <th>ucode</th>\n",
              "      <th>aliases</th>\n",
              "      <th>alt_title</th>\n",
              "      <th>aliases_ascii</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>#️⃣</td>\n",
              "      <td>symbols</td>\n",
              "      <td>keycap number sign</td>\n",
              "      <td>[hash]</td>\n",
              "      <td>0023-fe0f-20e3</td>\n",
              "      <td>[keycap, symbol, pound, hash, number]</td>\n",
              "      <td>0023_fe0f_20e3</td>\n",
              "      <td>#️⃣</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>*️⃣</td>\n",
              "      <td>symbols</td>\n",
              "      <td>keycap asterisk</td>\n",
              "      <td>[asterisk]</td>\n",
              "      <td>002a-fe0f-20e3</td>\n",
              "      <td>[keycap, symbol, asterisk, star]</td>\n",
              "      <td>002a_fe0f_20e3</td>\n",
              "      <td>*️⃣</td>\n",
              "      <td>[:keycap_asterisk:]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0️⃣</td>\n",
              "      <td>symbols</td>\n",
              "      <td>keycap digit zero</td>\n",
              "      <td>[zero]</td>\n",
              "      <td>0030-fe0f-20e3</td>\n",
              "      <td>[, keycap, symbol, number, zero, math]</td>\n",
              "      <td>0030_fe0f_20e3</td>\n",
              "      <td>0️⃣</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1️⃣</td>\n",
              "      <td>symbols</td>\n",
              "      <td>keycap digit one</td>\n",
              "      <td>[one]</td>\n",
              "      <td>0031-fe0f-20e3</td>\n",
              "      <td>[, keycap, symbol, number, one, math]</td>\n",
              "      <td>0031_fe0f_20e3</td>\n",
              "      <td>1️⃣</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2️⃣</td>\n",
              "      <td>symbols</td>\n",
              "      <td>keycap digit two</td>\n",
              "      <td>[two]</td>\n",
              "      <td>0032-fe0f-20e3</td>\n",
              "      <td>[, keycap, symbol, two, number, math]</td>\n",
              "      <td>0032_fe0f_20e3</td>\n",
              "      <td>2️⃣</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  Unnamed: 0 category               title      shorts unicode_alternates  \\\n",
              "0        #️⃣  symbols  keycap number sign      [hash]     0023-fe0f-20e3   \n",
              "1        *️⃣  symbols     keycap asterisk  [asterisk]     002a-fe0f-20e3   \n",
              "2        0️⃣  symbols   keycap digit zero      [zero]     0030-fe0f-20e3   \n",
              "3        1️⃣  symbols    keycap digit one       [one]     0031-fe0f-20e3   \n",
              "4        2️⃣  symbols    keycap digit two       [two]     0032-fe0f-20e3   \n",
              "\n",
              "                                 keywords     ucode_short ucode  \\\n",
              "0   [keycap, symbol, pound, hash, number]  0023_fe0f_20e3   #️⃣   \n",
              "1        [keycap, symbol, asterisk, star]  002a_fe0f_20e3   *️⃣   \n",
              "2  [, keycap, symbol, number, zero, math]  0030_fe0f_20e3   0️⃣   \n",
              "3   [, keycap, symbol, number, one, math]  0031_fe0f_20e3   1️⃣   \n",
              "4   [, keycap, symbol, two, number, math]  0032_fe0f_20e3   2️⃣   \n",
              "\n",
              "               aliases alt_title aliases_ascii  \n",
              "0                  NaN       NaN           NaN  \n",
              "1  [:keycap_asterisk:]       NaN           NaN  \n",
              "2                  NaN       NaN           NaN  \n",
              "3                  NaN       NaN           NaN  \n",
              "4                  NaN       NaN           NaN  "
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Size of emoji map:\", emoji_map.shape)\n",
        "emoji_map.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "UyxLni3AaUo9",
        "outputId": "3ef48909-f9ed-4045-9afd-bb9b8ded04f3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(1791, 11)"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "emoji_map.rename(columns={'Unnamed: 0': 'ucode1'}, inplace=True)\n",
        "emoji_map[(emoji_map['ucode1'] == emoji_map['ucode'])].shape\n",
        "# ucode and ucode1 are the same because the shapes are the same"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "tgHP4_Dem0fM",
        "outputId": "0f3ec0cb-0b8c-4a6c-8233-bf5d93639ec3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'😎'"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# emoji_map.loc[[768, 775, 520, 776, 525, 784, 785, 787, 1556, 791, 537, 1562, 539, 540, 796, 1310, 798, 544, 806, 554, 557, 1582, 815, 818, 1330, 566, 567, 56, 569, 58, 57, 824, 1598, 577, 1604, 585, 1354, 593, 1361, 1362, 1365, 1367, 600, 1119, 1125, 1126, 615, 103, 1389, 1134, 1144, 1406, 130, 642, 648, 649, 1162, 651, 652, 1167, 143, 1169, 913, 916, 662, 663, 664, 919, 920, 1180, 1437, 670, 926, 930, 681, 684, 174, 1200, 1201, 177, 1210, 1217, 200, 721, 1255, 1263, 754, 757, 509, 510], 'ucode']\n",
        "emoji_map.loc[1393, 'ucode']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5nYNXIe5YK9M"
      },
      "source": [
        "# Cleaning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BLWqY7vn4f1T"
      },
      "source": [
        "## Remove duplicate annotations (and store them in a sorted list)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9whRbWY2FPfc"
      },
      "source": [
        "For train_full, test_full, and valid_full, I think there are no duplicates and already in order. For the image, a lot of duplicates\n",
        "-- Clifton"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "e201ItAIAnV9"
      },
      "outputs": [],
      "source": [
        "remove_duplicates = lambda annotation : sorted(list(int(x) for x in set(annotation.split(\",\"))))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "WueWJls8_U_B"
      },
      "outputs": [],
      "source": [
        "test_full[\"annotations\"] = test_full[\"annotations\"].apply(remove_duplicates)\n",
        "train_full[\"annotations\"] = train_full[\"annotations\"].apply(remove_duplicates)\n",
        "valid_full[\"annotations\"] = valid_full[\"annotations\"].apply(remove_duplicates)\n",
        "\n",
        "test_img[\"annotations\"] = test_img[\"annotations\"].apply(remove_duplicates)\n",
        "train_img[\"annotations\"] = train_img[\"annotations\"].apply(remove_duplicates)\n",
        "valid_img[\"annotations\"] = valid_img[\"annotations\"].apply(remove_duplicates)\n",
        "\n",
        "test_balanced[\"annotations\"] = test_balanced[\"annotations\"].apply(remove_duplicates)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ahpfPuigIRcb"
      },
      "source": [
        "## Filter out Tweets with Images"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CIGSjrU3I2CC"
      },
      "source": [
        "Key Point:\n",
        "- There are some tweets in train_img but not in train_full (e.g. 760790155389243393)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aUOVyU7-DyOi",
        "outputId": "400cb0b5-9e30-48e9-d7fe-4ff7295f1637"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total training rows without imgid before: 12997219\n",
            "Total training rows with imgid before: 916864\n",
            "Total training rows without imgid after: 12124064\n",
            "Total training rows with imgid after: 43709\n",
            "Total rows in train_text final: 12124064\n"
          ]
        }
      ],
      "source": [
        "train_text = pd.concat([train_full, train_img])\n",
        "print(\"Total training rows without imgid before: \" + str(len(train_text[train_text[\"imgid\"].isnull()])))\n",
        "print(\"Total training rows with imgid before: \" + str(len(train_text[train_text[\"imgid\"].notnull()])))\n",
        "\n",
        "train_text = train_text.drop_duplicates(subset = 'id', keep = False)\n",
        "print(\"Total training rows without imgid after: \" + str(len(train_text[train_text[\"imgid\"].isnull()])))\n",
        "print(\"Total training rows with imgid after: \" + str(len(train_text[train_text[\"imgid\"].notnull()])))\n",
        "\n",
        "train_text = train_text[train_text[\"imgid\"].isnull()]\n",
        "train_text = train_text.drop('imgid', axis=1)\n",
        "print(\"Total rows in train_text final: \" + str(len(train_text)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nr-SXAWXJIZL",
        "outputId": "e6fbc7bc-3335-4620-eeb2-68579ee217e7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total test rows without imgid before: 1000000\n",
            "Total test rows with imgid before: 79191\n",
            "Total test rows without imgid after: 932573\n",
            "Total test rows with imgid after: 11764\n",
            "Total test rows in test_text final: 932573\n"
          ]
        }
      ],
      "source": [
        "test_text = pd.concat([test_full, test_img])\n",
        "print(\"Total test rows without imgid before: \" + str(len(test_text[test_text[\"imgid\"].isnull()])))\n",
        "print(\"Total test rows with imgid before: \" + str(len(test_text[test_text[\"imgid\"].notnull()])))\n",
        "\n",
        "test_text = test_text.drop_duplicates(subset = 'id', keep = False)\n",
        "print(\"Total test rows without imgid after: \" + str(len(test_text[test_text[\"imgid\"].isnull()])))\n",
        "print(\"Total test rows with imgid after: \" + str(len(test_text[test_text[\"imgid\"].notnull()])))\n",
        "\n",
        "test_text = test_text[test_text[\"imgid\"].isnull()]\n",
        "test_text = test_text.drop('imgid', axis=1)\n",
        "print(\"Total test rows in test_text final: \" + str(len(test_text)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "atAaB_f1MKtM",
        "outputId": "bd19fd3e-d5a2-44f7-8812-3da98f9b0c2d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total validation rows without imgid before: 1000000\n",
            "Total validation rows with imgid before: 79419\n",
            "Total validation rows without imgid after: 932533\n",
            "Total validation rows with imgid after: 11952\n",
            "Total validation rows in valid_text final: 932533\n"
          ]
        }
      ],
      "source": [
        "valid_text = pd.concat([valid_full, valid_img])\n",
        "print(\"Total validation rows without imgid before: \" + str(len(valid_text[valid_text[\"imgid\"].isnull()])))\n",
        "print(\"Total validation rows with imgid before: \" + str(len(valid_text[valid_text[\"imgid\"].notnull()])))\n",
        "\n",
        "valid_text = valid_text.drop_duplicates(subset = 'id', keep = False)\n",
        "print(\"Total validation rows without imgid after: \" + str(len(valid_text[valid_text[\"imgid\"].isnull()])))\n",
        "print(\"Total validation rows with imgid after: \" + str(len(valid_text[valid_text[\"imgid\"].notnull()])))\n",
        "\n",
        "valid_text = valid_text[valid_text[\"imgid\"].isnull()]\n",
        "valid_text = valid_text.drop('imgid', axis=1)\n",
        "print(\"Total validation rows in valid_text final: \" + str(len(valid_text)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "epq5RR78O870",
        "outputId": "f10b6b86-e017-4c0d-ca4b-7e3f01977182"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>742407819496919041</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744101567981359105</td>\n",
              "      <td>[1421]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>744692442033577984</td>\n",
              "      <td>[1056]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>746426733834944512</td>\n",
              "      <td>[1056, 1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>744959564504793088_q</td>\n",
              "      <td>[616]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997214</th>\n",
              "      <td>744270931997171712</td>\n",
              "      <td>[1210]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997215</th>\n",
              "      <td>743546441591721984</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997216</th>\n",
              "      <td>742894986304344064</td>\n",
              "      <td>[12, 114, 1111]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997217</th>\n",
              "      <td>741607748975132672</td>\n",
              "      <td>[1428]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997218</th>\n",
              "      <td>744088860909518848</td>\n",
              "      <td>[1107]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>12124064 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                            id      annotations\n",
              "1           742407819496919041           [1381]\n",
              "2           744101567981359105           [1421]\n",
              "3           744692442033577984           [1056]\n",
              "4           746426733834944512     [1056, 1381]\n",
              "5         744959564504793088_q            [616]\n",
              "...                        ...              ...\n",
              "12997214    744270931997171712           [1210]\n",
              "12997215    743546441591721984           [1447]\n",
              "12997216    742894986304344064  [12, 114, 1111]\n",
              "12997217    741607748975132672           [1428]\n",
              "12997218    744088860909518848           [1107]\n",
              "\n",
              "[12124064 rows x 2 columns]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "ague0iojPAZM",
        "outputId": "b10685b7-020d-45a6-d26d-bc38652c0987"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>747643690521341953</td>\n",
              "      <td>[1138]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>746196870121164800</td>\n",
              "      <td>[1056]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744595559470800896</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>747720844730859520</td>\n",
              "      <td>[889, 1128]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>743285629576843264</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999995</th>\n",
              "      <td>742645704074481664</td>\n",
              "      <td>[1105]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999996</th>\n",
              "      <td>747015502791335936</td>\n",
              "      <td>[1400]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999997</th>\n",
              "      <td>744745632141737984</td>\n",
              "      <td>[807]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999998</th>\n",
              "      <td>747071768817471488</td>\n",
              "      <td>[1389]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999999</th>\n",
              "      <td>748393225166127105</td>\n",
              "      <td>[1393, 1447]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>932573 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        id   annotations\n",
              "0       747643690521341953        [1138]\n",
              "1       746196870121164800        [1056]\n",
              "2       744595559470800896        [1381]\n",
              "3       747720844730859520   [889, 1128]\n",
              "4       743285629576843264        [1424]\n",
              "...                    ...           ...\n",
              "999995  742645704074481664        [1105]\n",
              "999996  747015502791335936        [1400]\n",
              "999997  744745632141737984         [807]\n",
              "999998  747071768817471488        [1389]\n",
              "999999  748393225166127105  [1393, 1447]\n",
              "\n",
              "[932573 rows x 2 columns]"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "jqqVsSE8PBuW",
        "outputId": "220a7fd3-ceca-4620-f82f-f1386cdfd4fb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>746634523048292352</td>\n",
              "      <td>[1103, 1389, 1399]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>741650544847147009</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>746803092239196162</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>746625375875170304</td>\n",
              "      <td>[1107]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>744724790485757952</td>\n",
              "      <td>[1620]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999995</th>\n",
              "      <td>747420757790162944</td>\n",
              "      <td>[1620]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999996</th>\n",
              "      <td>744673950962225152</td>\n",
              "      <td>[544]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999997</th>\n",
              "      <td>741743895000809472</td>\n",
              "      <td>[1415]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999998</th>\n",
              "      <td>745039072628899841</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999999</th>\n",
              "      <td>746762117898895361</td>\n",
              "      <td>[186]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>932533 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        id         annotations\n",
              "0       746634523048292352  [1103, 1389, 1399]\n",
              "1       741650544847147009              [1381]\n",
              "2       746803092239196162              [1424]\n",
              "3       746625375875170304              [1107]\n",
              "4       744724790485757952              [1620]\n",
              "...                    ...                 ...\n",
              "999995  747420757790162944              [1620]\n",
              "999996  744673950962225152               [544]\n",
              "999997  741743895000809472              [1415]\n",
              "999998  745039072628899841              [1447]\n",
              "999999  746762117898895361               [186]\n",
              "\n",
              "[932533 rows x 2 columns]"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "valid_text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Remove Tweets with A Quoted Tweet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "J8rBVXFTjcRX"
      },
      "outputs": [],
      "source": [
        "test_text = test_text[test_text['id'].str[-2:] != \"_q\"]\n",
        "train_text = train_text[train_text['id'].str[-2:] != \"_q\"]\n",
        "valid_text = valid_text[valid_text['id'].str[-2:] != \"_q\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "4B7dC0Tzq5fK"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>747643690521341953</td>\n",
              "      <td>[1138]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>746196870121164800</td>\n",
              "      <td>[1056]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744595559470800896</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>747720844730859520</td>\n",
              "      <td>[889, 1128]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>743285629576843264</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999995</th>\n",
              "      <td>742645704074481664</td>\n",
              "      <td>[1105]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999996</th>\n",
              "      <td>747015502791335936</td>\n",
              "      <td>[1400]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999997</th>\n",
              "      <td>744745632141737984</td>\n",
              "      <td>[807]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999998</th>\n",
              "      <td>747071768817471488</td>\n",
              "      <td>[1389]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999999</th>\n",
              "      <td>748393225166127105</td>\n",
              "      <td>[1393, 1447]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>879091 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        id   annotations\n",
              "0       747643690521341953        [1138]\n",
              "1       746196870121164800        [1056]\n",
              "2       744595559470800896        [1381]\n",
              "3       747720844730859520   [889, 1128]\n",
              "4       743285629576843264        [1424]\n",
              "...                    ...           ...\n",
              "999995  742645704074481664        [1105]\n",
              "999996  747015502791335936        [1400]\n",
              "999997  744745632141737984         [807]\n",
              "999998  747071768817471488        [1389]\n",
              "999999  748393225166127105  [1393, 1447]\n",
              "\n",
              "[879091 rows x 2 columns]"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "uWnW0AXyq7cn"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>742407819496919041</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744101567981359105</td>\n",
              "      <td>[1421]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>744692442033577984</td>\n",
              "      <td>[1056]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>746426733834944512</td>\n",
              "      <td>[1056, 1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>743951261607690240</td>\n",
              "      <td>[1387]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997214</th>\n",
              "      <td>744270931997171712</td>\n",
              "      <td>[1210]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997215</th>\n",
              "      <td>743546441591721984</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997216</th>\n",
              "      <td>742894986304344064</td>\n",
              "      <td>[12, 114, 1111]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997217</th>\n",
              "      <td>741607748975132672</td>\n",
              "      <td>[1428]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997218</th>\n",
              "      <td>744088860909518848</td>\n",
              "      <td>[1107]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11433540 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                          id      annotations\n",
              "1         742407819496919041           [1381]\n",
              "2         744101567981359105           [1421]\n",
              "3         744692442033577984           [1056]\n",
              "4         746426733834944512     [1056, 1381]\n",
              "6         743951261607690240           [1387]\n",
              "...                      ...              ...\n",
              "12997214  744270931997171712           [1210]\n",
              "12997215  743546441591721984           [1447]\n",
              "12997216  742894986304344064  [12, 114, 1111]\n",
              "12997217  741607748975132672           [1428]\n",
              "12997218  744088860909518848           [1107]\n",
              "\n",
              "[11433540 rows x 2 columns]"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "hXXTmvDkq8d3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>746634523048292352</td>\n",
              "      <td>[1103, 1389, 1399]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>741650544847147009</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>746803092239196162</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>746625375875170304</td>\n",
              "      <td>[1107]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>744724790485757952</td>\n",
              "      <td>[1620]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999995</th>\n",
              "      <td>747420757790162944</td>\n",
              "      <td>[1620]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999996</th>\n",
              "      <td>744673950962225152</td>\n",
              "      <td>[544]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999997</th>\n",
              "      <td>741743895000809472</td>\n",
              "      <td>[1415]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999998</th>\n",
              "      <td>745039072628899841</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999999</th>\n",
              "      <td>746762117898895361</td>\n",
              "      <td>[186]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>879201 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        id         annotations\n",
              "0       746634523048292352  [1103, 1389, 1399]\n",
              "1       741650544847147009              [1381]\n",
              "2       746803092239196162              [1424]\n",
              "3       746625375875170304              [1107]\n",
              "4       744724790485757952              [1620]\n",
              "...                    ...                 ...\n",
              "999995  747420757790162944              [1620]\n",
              "999996  744673950962225152               [544]\n",
              "999997  741743895000809472              [1415]\n",
              "999998  745039072628899841              [1447]\n",
              "999999  746762117898895361               [186]\n",
              "\n",
              "[879201 rows x 2 columns]"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "valid_text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Observing emoji frequency distribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1381 😂\n",
            "1424 😭\n",
            "1392 😍\n",
            "1447 🙄\n",
            "186 ❤\n",
            "1389 😊\n",
            "1420 😩\n",
            "1620 🤔\n",
            "1403 😘\n",
            "763 🏽\n",
            "762 🏼\n",
            "764 🏾\n",
            "1138 💯\n",
            "1107 💕\n",
            "1446 🙃\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6IAAAITCAYAAAAdGaHjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAif0lEQVR4nO3de5RlZ1kn4N9LmjtykTSzkAQ6YlCjIpeeeJvRCI6G4CLgqEMUFAfM0hFFATUMs5DBcYaLI15AmYgIKBIQFTMSwRsIgwTT3EIugk2I0pExzUURGQzRb/7Yu+GkUt1V3X3qra7D86xVK2d/+zunvre/XTv7d/Y++9QYIwAAANDlFts9AAAAAD67CKIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAq20NolX1oqq6vqqu2GT/b6+qq6rqyqr6ja0eHwAAAMtX2/k9olX1tUk+nuSlY4wv3aDv6UlemeRBY4yPVtXdxhjXd4wTAACA5dnWM6JjjDcm+chiW1Xdu6peW1Vvq6o3VdUXzau+N8nzxxgfnZ8rhAIAAOxAJ+JnRC9M8oNjjAcmeXKSX5zb75PkPlX15qq6tKrO3rYRAgAAcMx2bfcAFlXVHZJ8dZLfrKpDzbee/7sryelJzkpySpI3VtWXjTH+rnmYAAAAHIcTKohmOkP7d2OM+62z7kCSt44xPpXk/VX13kzB9LLG8QEAAHCcTqhLc8cYH8sUMr8tSWry5fPqV2c6G5qqOjnTpbrXbMMwAQAAOA7b/fUtL0/yliRfWFUHquqxSb4zyWOr6l1Jrkxy7tz9dUk+XFVXJXl9kh8dY3x4O8YNAADAsdvWr28BAADgs88JdWkuAAAAq08QBQAAoNW23TX35JNPHnv27NmuXw8AAMAWetvb3vahMcbu9dZtWxDds2dP9u3bt12/HgAAgC1UVX91uHUuzQUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFrt2u4BnMj2XPCa7R7CUbn2mQ/d7iEAAABsyBlRAAAAWgmiAAAAtNowiFbVi6rq+qq64jDrv7OqLq+qd1fVn1XVly9/mAAAAKyKzZwRfXGSs4+w/v1Jvm6M8WVJfjLJhUsYFwAAACtqw5sVjTHeWFV7jrD+zxYWL01yyhLGBQAAwIpa9mdEH5vk9w+3sqrOr6p9VbXv4MGDS/7VAAAA7ARLC6JV9fWZguiPH67PGOPCMcbeMcbe3bt3L+tXAwAAsIMs5XtEq+q+SV6Y5CFjjA8v4zUBAABYTcd9RrSq7pnkt5M8eozx3uMfEgAAAKtswzOiVfXyJGclObmqDiT5iSS3TJIxxguSPC3JXZP8YlUlyY1jjL1bNWAAAAB2ts3cNfe8DdY/LsnjljYiAAAAVtqy75oLAAAARySIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACg1YZBtKpeVFXXV9UVh1lfVfXzVbW/qi6vqgcsf5gAAACsis2cEX1xkrOPsP4hSU6ff85P8kvHPywAAABW1YZBdIzxxiQfOUKXc5O8dEwuTXLnqrr7sgYIAADAalnGZ0TvkeQDC8sH5jYAAAC4mdabFVXV+VW1r6r2HTx4sPNXAwAAcIJYRhC9LsmpC8unzG03M8a4cIyxd4yxd/fu3Uv41QAAAOw0ywiiFyf5rvnuuV+Z5O/HGB9cwusCAACwgnZt1KGqXp7krCQnV9WBJD+R5JZJMsZ4QZJLkpyTZH+STyT5nq0aLAAAADvfhkF0jHHeButHkh9Y2ogAAABYaa03KwIAAABBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWm0qiFbV2VX1nqraX1UXrLP+nlX1+qp6R1VdXlXnLH+oAAAArIINg2hVnZTk+UkekuSMJOdV1Rlruv2XJK8cY9w/ySOT/OKyBwoAAMBq2MwZ0TOT7B9jXDPGuCHJRUnOXdNnJLnj/PhOSf5meUMEAABglWwmiN4jyQcWlg/MbYuenuRRVXUgySVJfnC9F6qq86tqX1XtO3jw4DEMFwAAgJ1uWTcrOi/Ji8cYpyQ5J8mvVdXNXnuMceEYY+8YY+/u3buX9KsBAADYSTYTRK9LcurC8ilz26LHJnllkowx3pLkNklOXsYAAQAAWC2bCaKXJTm9qk6rqltluhnRxWv6/HWSBydJVX1xpiDq2lsAAABuZsMgOsa4Mcnjk7wuydWZ7o57ZVU9o6oeNnd7UpLvrap3JXl5kseMMcZWDRoAAICda9dmOo0xLsl0E6LFtqctPL4qydcsd2gAAACsomXdrAgAAAA2RRAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWmwqiVXV2Vb2nqvZX1QWH6fPtVXVVVV1ZVb+x3GECAACwKnZt1KGqTkry/CT/LsmBJJdV1cVjjKsW+pye5ClJvmaM8dGquttWDRgAAICdbTNnRM9Msn+Mcc0Y44YkFyU5d02f703y/DHGR5NkjHH9cocJAADAqthMEL1Hkg8sLB+Y2xbdJ8l9qurNVXVpVZ293gtV1flVta+q9h08ePDYRgwAAMCOtqybFe1KcnqSs5Kcl+SXq+rOazuNMS4cY+wdY+zdvXv3kn41AAAAO8lmguh1SU5dWD5lblt0IMnFY4xPjTHen+S9mYIpAAAA3MRmguhlSU6vqtOq6lZJHpnk4jV9Xp3pbGiq6uRMl+pes7xhAgAAsCo2DKJjjBuTPD7J65JcneSVY4wrq+oZVfWwudvrkny4qq5K8vokPzrG+PBWDRoAAICda8Ovb0mSMcYlSS5Z0/a0hccjyRPnHwAAADisZd2sCAAAADZFEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALTatd0DYHvsueA12z2ETbv2mQ/d7iEAAABL5IwoAAAArQRRAAAAWgmiAAAAtBJEAQAAaOVmRayUnXQTpsSNmAAA+OzkjCgAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACgla9vgR3CV9MAALAqnBEFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtPL1LcC289U0AACfXTZ1RrSqzq6q91TV/qq64Aj9/n1Vjarau7whAgAAsEo2DKJVdVKS5yd5SJIzkpxXVWes0+9zkjwhyVuXPUgAAABWx2bOiJ6ZZP8Y45oxxg1JLkpy7jr9fjLJs5J8conjAwAAYMVsJojeI8kHFpYPzG2fVlUPSHLqGGNnfdALAACAdsd919yqukWSn0nypE30Pb+q9lXVvoMHDx7vrwYAAGAH2kwQvS7JqQvLp8xth3xOki9N8oaqujbJVya5eL0bFo0xLhxj7B1j7N29e/exjxoAAIAdazNB9LIkp1fVaVV1qySPTHLxoZVjjL8fY5w8xtgzxtiT5NIkDxtj7NuSEQMAALCjbRhExxg3Jnl8ktcluTrJK8cYV1bVM6rqYVs9QAAAAFbLrs10GmNckuSSNW1PO0zfs45/WAAAAKyq475ZEQAAABwNQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWu3a7gEArLI9F7xmu4dwVK595kO3ewgAwGcBZ0QBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWvkeUQCOie9IBQCOlTOiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFa+vgUA1vDVNACwtZwRBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWu3a7gEAAH32XPCa7R7Cpl37zIduuu9Oqis5utoAVpEzogAAALQSRAEAAGgliAIAANBKEAUAAKCVmxUBAJzA3IgJWEXOiAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACt3DUXAIBt4Y7A8NlLEAUAgCUTsuHIXJoLAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtHLXXAAAYNNW+Y7AO6m2nX6n402dEa2qs6vqPVW1v6ouWGf9E6vqqqq6vKr+uKrutfyhAgAAsAo2DKJVdVKS5yd5SJIzkpxXVWes6faOJHvHGPdN8qokz172QAEAAFgNmzkjemaS/WOMa8YYNyS5KMm5ix3GGK8fY3xiXrw0ySnLHSYAAACrYjNB9B5JPrCwfGBuO5zHJvn99VZU1flVta+q9h08eHDzowQAAGBlLPWuuVX1qCR7kzxnvfVjjAvHGHvHGHt37969zF8NAADADrGZu+Zel+TUheVT5rabqKpvSPLUJF83xvin5QwPAACAVbOZM6KXJTm9qk6rqlsleWSSixc7VNX9k/yvJA8bY1y//GECAACwKjYMomOMG5M8Psnrklyd5JVjjCur6hlV9bC523OS3CHJb1bVO6vq4sO8HAAAAJ/lNnNpbsYYlyS5ZE3b0xYef8OSxwUAAMCKWurNigAAAGAjgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFoJogAAALQSRAEAAGgliAIAANBKEAUAAKCVIAoAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACglSAKAABAK0EUAACAVoIoAAAArQRRAAAAWgmiAAAAtBJEAQAAaCWIAgAA0EoQBQAAoJUgCgAAQCtBFAAAgFaCKAAAAK0EUQAAAFptKohW1dlV9Z6q2l9VF6yz/tZV9Yp5/Vuras/SRwoAAMBK2DCIVtVJSZ6f5CFJzkhyXlWdsabbY5N8dIzxBUmem+RZyx4oAAAAq2EzZ0TPTLJ/jHHNGOOGJBclOXdNn3OTvGR+/KokD66qWt4wAQAAWBWbCaL3SPKBheUDc9u6fcYYNyb5+yR3XcYAAQAAWC01xjhyh6pvTXL2GONx8/Kjk3zFGOPxC32umPscmJffN/f50JrXOj/J+fPiFyZ5z7IK2WFOTvKhDXvtTKta26rWlahtp1LbzrSqta1qXYnadiq17UyrWtuq1rUZ9xpj7F5vxa5NPPm6JKcuLJ8yt63X50BV7UpypyQfXvtCY4wLk1y4mRGvsqraN8bYu93j2AqrWtuq1pWobadS2860qrWtal2J2nYqte1Mq1rbqtZ1vDZzae5lSU6vqtOq6lZJHpnk4jV9Lk7y3fPjb03yJ2OjU60AAAB8VtrwjOgY48aqenyS1yU5KcmLxhhXVtUzkuwbY1yc5FeS/FpV7U/ykUxhFQAAAG5mM5fmZoxxSZJL1rQ9beHxJ5N823KHttJW+fLkVa1tVetK1LZTqW1nWtXaVrWuRG07ldp2plWtbVXrOi4b3qwIAAAAlmkznxEFAACApRFEj1NV/Y+q+vqqenhVPWVu+7aqurKq/qWq9i70PbOq3jn/vKuqHrGw7kfm51xRVS+vqtvM7Y+vqv1VNarq5BO1toXn3LOqPl5VT56XT62q11fVVfPznrDOc57UWd8S5+wJ83xdWVU/vND+nKr6i6q6vKp+p6ru3FHX0da28JybzNlC+0lV9Y6q+r2Ftjct/Hv8TVW9esuLWsdRzuEtq+olVfXuqrr6UP953Z2r6lXzfF1dVV+1A+o5lv3Ig6rq7XP7S2q6u/kJV9vCcza9H6mqz62qP6yqv5z/e5cTsa6qum9VvWVe/+6quk1V3a6qXjNvf1dW1TMX+t+6ql5R0/7/rVW1p6OuY6ltXn+z/UhVnV1V75lruGCh/Vfmbffy+e/vDj2V3WzM69X5ioW/r2ur6p0L/W82h3P7a+d6rqyqF1TVSdtRz6Jl1Hak7XM7LWveFtZfXNPXEG6Lo/l7q6q7zvvCj1fV89a8zrrbYVXdr6ounf9t9lXVmTuwtgfOc7e/qn6+qmpuP+y8nyh1LTznaI61qqp+qqreW9OxyQ9tfVXbZIzh5zh+kvxJktsmeW6Sr5nbvjjT96S+Icnehb63S7Jrfnz3JNdn+pzuPZK8P8lt53WvTPKY+fH9k+xJcm2Sk0/U2hae86okv5nkyQt1PmB+/DlJ3pvkjIX+p2a6EdZfddW3pDn70iRXHFqf5I+SfMHc7xsXnvOsJM/aSXO20P7EJL+R5PcO87t+K8l3dW6TxziH35HkooX5vDbJnnn5JUkeNz++VZI774B6jmo/kukNxw8kuc/c/owkjz0RazvcNpkj7EeSPDvJBfPjC7r+3o5yznYluTzJl8/Ld81087/bJfn6he3vTUkeMi//pyQvmB8/MskrdticnZTkfUk+f67tXQtzdseF5/3Mofnr/lmvzjXr/2eSpx1pDhfrSVKZ9ouP3I56ll3bkbbPnV7bQt9vyfT/uStOpHqOsC+5fZJ/k+T7kjxvzeusux0m+YOF/co5Sd6wA2v78yRfOdf2++tth4vzfiLVtfCcTR9rJfmeJC9Ncot5+W7btX1u9U/bu+Krpqqek+SbkpyW5C1J7p3kwVX1qjHGM+Y+N3nOGOMTC4u3SbL4Ad1dSW5bVZ/KtPP/m/k571jvtbbSsdQ2tz0804HwPx5qG2N8MMkH58f/UFVXZzpgvmru8twkP5bkd7eonMXxLXPOvjjJWw+tr6o/zfQ/tGePMf5g4TmXZvpKoy21zDmb209J8tAkP5VpJ7n2eXdM8qBMO8s2x1jnSHL7ms4C3jbJDUk+VlV3SvK1mcJaxhg3zOvaNO1H7prkhjHGe+c+f5jkKZnudr5lGvcj5yY5a+76kkwHAT++BSUdGt+x1PWNSS4fY7xrruHQ92x/Isnr57Ybqurtmb6rO5nqevr8+FVJnldVNeajkq2w5P3ImUn2jzGumftclKmmq8YYH5vbKtPfZOvNKjZZZyX59kz7ueTwc5hD9WT6+7tVmutZtOTajrR9tlv2vNV0Jv6JSc7P9MZdq2Pc//9jkv9TVV+w9vWOsB2OJHecH98p8/HlVlpmbVV190wh+9J5+aVJHp4pkB7qs3bet0Tjsdb3J/mOMca/JMkY4/pl13LC2O4kvJN/kvzrJL+Q5JZJ3rzO+jdkzbsiSb4iyZVJPp7kEQvtT5jbDiZ52TqvdW0az4gebW1J7pDpj/IOmQ6enrzOc/Yk+et85l27c5P8XGd9y5qzTEH0vZkO8m831/4L67ze/07yqJ02Z5kOfB+Y6QD/ZmdEk3xXkld1bY/HWectk1w0/239Y5Lz5/b7ZXqX9cVJ3pHkhUluf6LXc7htcm6/2X4k0zvIf3XoNZL8XJJ3n4i1HeN+5O8W1tXi8glU1w8n+bVMV3+8PcmPrfOcOye5Jsnnz8tXJDllYf37cgLuIw83Z5negHvhQr9HZ+EsR5JfTfK3mYLO7Tq2x6Os82szfUXdpuZwbv9opjMbJ23l2LtrW2/7XIXaMr0R/oh5n7ItZ0SP9u9tof0xWXPW8HDbYabjlb/OdGXMdUnutZNqS7I3yR8tLP/brDkuWTvvJ1JdOYZjrSQfTvLUJPsyBe7Tt2P77PjxGdHj84BMlxt9UZKrN/OEMcZbxxhfkmlDfkpNn8O4S6ZQdlqSz8t09uZRWzTmzTra2p6e5LljjI+vt3J+5/G3kvzwGONjVXW7JP85ydPW67+FljJnY4yrM112+wdJXpvknUn+efF5VfXUJDcmednSRn9kS5mzqvrmJNePMd52hOeel+TlxzjO43W0dZ6ZaW4+L9Pf2JOq6vMzvWv8gCS/NMa4f6aQesFhX2XrbOl+ZEz/V3tkkudW1Z8n+Yes2Va30JbuR9aun2vtOBt1tHXtynTJ2XfO/31EVT340Mr5bP3Lk/z8mM8gbqOlztnhjDG+J9N2enWS/3CUY1yGjepcu4874hyOMb4p0yXkt84Wn5XZhKXWtsO2z03VVlX3S3LvMcbvbO1wN3TU+/8jOcx2+P1JfmSMcWqSH8kWXw2zYKm1baDzmKTjWOvWST45xtib5JeTvOi4RnwCc2nuMZh3YC/OdInKhzKdEauaPiT9VWOM/7fRa4wxrq6qj2f6rOFpSd4/xjg4v/5vJ/nqJL++FeM/kuOo7SuSfGtVPTvTO6f/UlWfHGM8r6pumeng8WVjjN+e+987U93vmi9jOCXJ26vqzDHG/z2B6vq0NXO2b4zxK5l36FX135McWPh9j0nyzUkePB8cb5llz1mmSx4fVlXnZLr0845V9etjjEfNv+/kTOHuEeu/7NY4jjq/I8lrxxifSnJ9Vb050zusb0xyYIzx1rnfq9IYRDv3I2OMt2R6FzlV9Y1J7rPsehY17keS5G+r6u5jjA/Ol3Bt2SVMx1HXgSRvHGN8aH6dSzIdzPzxvP7CJH85xvjZhedcl+kz9AfmIHCnTO+Sb4kt2I+8bR7/IadkqunTxhj/PF+y+2OZzpBuuc3UOf97f0umMxWHbDSHGWN8sqp+N9MbQn+45cWssYW1rbd9ttqC2j6eZG9VXZvpWPhuVfWGMcZZJ0o9x/ra62yH353paplk+oziC4954JuwBbVdl5teEn6Tfclh5n3pmo+1DiQ59P+530nT/nFbbPcp2Z38k+TPMt0I5MVZuAHPwvo35Kan50/LZ24ycq9M1+mfnM9cZne7TJeWvSTJD655rWvTe2nuUdW2Zt3T85nLsyrTB65/doPf11LfsuZsXr7b/N97JvmLzDe6SXJ2ps+u7T6Rt8fDzdma9rNy80tgvi/JSzprO845/PEkvzo/vv08N/edl9+U5AsX/g2eswPqOer9yMK2eutMB5cPOhFrO9w2eaT9SJLn5KY3K3r2iVZXkrtkuixw8eZmD53X/bdMAfsWa17jB3LTmxW9cofN2a5Ml3Kels/crOhL5rk8dGO3SvLTSX66o7bN1plpH/6na9rWncNMl9vdfaHmVyR5fHc9W1HbkbbPVahtoc+ebN+lucf095abX7562O0w01m7s+bHD07ytp1U29y29mZF5xxp3k/EuuZ1T88mjrWSPDPJf1xYd9l2bJ8t/57bPYCd+pNkd5LXzI8vXbPuEZnezfinTJ+Bed3c/uhMB4rvnHeMD194zn/NFGauyPR5hlvP7T80v9aNmQ44X3gi1ramz6f/0DJdCjMy3bXunfPPOes859pscRDdgjl7U6ZQ865MZz4Pte/P9FmMQ/W+YCfN2Zr2m+wc57Y3JDl7q2ta4hzeIdO7wFfO8/WjC8+5X6bPYFye5NVJ7rID6jmW/chzMh2MvCfTZa0nZG2H2yZzhP1Ips9p/3GSv8x0kPm5J2JdSR41z9sVmcNypnfWxzw3h+o6dBfn28zb7f5MB2Fb/tm8Zc7ZvHxOps/Svy/JU+e2WyR5c5J3z/8WL8vCXXS3e9uc216c5PvWaV9vDv9VksvmbfOKTJ8d29VZzxbWdtjtc6fXtmb9nmxDED2Ofcm1ST6S6azugSRnHGk7zLT/fFumY5W3JnngTqptbt871/W+JM9LUhvN+4lU10Kfp2dzQfTOSV6TaT/5lsx3fV7Fn5oLBgAAgBZuVgQAAEArQRQAAIBWgigAAACtBFEAAABaCaIAAAC0EkQBAABoJYgCAADQShAFAACg1f8HUpbjDRggmgcAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 1152x648 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "freq_dict = {}\n",
        "\n",
        "def count_freq(annotations):\n",
        "    for i in annotations:\n",
        "        freq_dict[i] = freq_dict.get(i, 0) + 1\n",
        "\n",
        "train_text[\"annotations\"].apply(count_freq)\n",
        "\n",
        "freq_dict = dict(sorted(freq_dict.items(), key=lambda x: x[1], reverse=True)[:15])\n",
        "\n",
        "for emoji_number in freq_dict.keys():\n",
        "    print(emoji_number, emoji_map.loc[int(emoji_number), \"ucode\"])\n",
        "\n",
        "ucode_dict = {}\n",
        "for emoji_number in freq_dict.keys():\n",
        "    ucode_dict[f\"#{emoji_number}\"] = freq_dict[emoji_number]\n",
        "\n",
        "plt.figure(figsize=(16, 9))\n",
        "plt.bar(*zip(*ucode_dict.items()))\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Remove weird emojis and keep the 12 remaining."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ucode1</th>\n",
              "      <th>title</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1381</th>\n",
              "      <td>😂</td>\n",
              "      <td>face with tears of joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1424</th>\n",
              "      <td>😭</td>\n",
              "      <td>loudly crying face</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1392</th>\n",
              "      <td>😍</td>\n",
              "      <td>smiling face with heart shaped eyes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1447</th>\n",
              "      <td>🙄</td>\n",
              "      <td>face with rolling eyes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>186</th>\n",
              "      <td>❤</td>\n",
              "      <td>heavy black heart</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1389</th>\n",
              "      <td>😊</td>\n",
              "      <td>smiling face with smiling eyes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1420</th>\n",
              "      <td>😩</td>\n",
              "      <td>weary face</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1620</th>\n",
              "      <td>🤔</td>\n",
              "      <td>thinking face</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1403</th>\n",
              "      <td>😘</td>\n",
              "      <td>face throwing a kiss</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>763</th>\n",
              "      <td>🏽</td>\n",
              "      <td>emoji modifier fitzpatrick type</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1138</th>\n",
              "      <td>💯</td>\n",
              "      <td>hundred points symbol</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1446</th>\n",
              "      <td>🙃</td>\n",
              "      <td>upside down face</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     ucode1                                title\n",
              "1381      😂               face with tears of joy\n",
              "1424      😭                   loudly crying face\n",
              "1392      😍  smiling face with heart shaped eyes\n",
              "1447      🙄               face with rolling eyes\n",
              "186       ❤                    heavy black heart\n",
              "1389      😊       smiling face with smiling eyes\n",
              "1420      😩                           weary face\n",
              "1620      🤔                        thinking face\n",
              "1403      😘                 face throwing a kiss\n",
              "763       🏽      emoji modifier fitzpatrick type\n",
              "1138      💯                hundred points symbol\n",
              "1446      🙃                     upside down face"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "emoji_list_final = list(freq_dict.keys())\n",
        "emoji_list_final.remove(762)\n",
        "emoji_list_final.remove(764)\n",
        "emoji_list_final.remove(1107)\n",
        "emoji_map_final = emoji_map.iloc[emoji_list_final, :]\n",
        "emoji_map_final.iloc[:,[0,2]]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[1381, 1424, 1392, 1447, 186, 1389, 1420, 1620, 1403, 763, 1138, 1446]"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "emoji_list_final"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Only keep the top 12 emoji as part of the labels."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {},
      "outputs": [],
      "source": [
        "def label(annotations):\n",
        "    if annotations:\n",
        "        lst = []\n",
        "        for i in annotations:\n",
        "            if i in emoji_list_final:\n",
        "                lst.append(i)\n",
        "        if lst:\n",
        "            return lst\n",
        "        else:\n",
        "            return"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(11433540, 2)"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pd.set_option('mode.chained_assignment', None) # suppress SettingCopyWarning\n",
        "train_text[\"annotations\"] = train_text[\"annotations\"].apply(label)\n",
        "train_text.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>742407819496919041</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>746426733834944512</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>747973915104972804</td>\n",
              "      <td>[763]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>741489131524493312</td>\n",
              "      <td>[1389]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>743594091108274180</td>\n",
              "      <td>[763]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997202</th>\n",
              "      <td>743063847964053505</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997204</th>\n",
              "      <td>741479246615433216</td>\n",
              "      <td>[1446]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997209</th>\n",
              "      <td>743318311333572608</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997211</th>\n",
              "      <td>742068066952065025</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12997215</th>\n",
              "      <td>743546441591721984</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4619330 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                          id annotations\n",
              "1         742407819496919041      [1381]\n",
              "4         746426733834944512      [1381]\n",
              "7         747973915104972804       [763]\n",
              "9         741489131524493312      [1389]\n",
              "13        743594091108274180       [763]\n",
              "...                      ...         ...\n",
              "12997202  743063847964053505      [1424]\n",
              "12997204  741479246615433216      [1446]\n",
              "12997209  743318311333572608      [1447]\n",
              "12997211  742068066952065025      [1447]\n",
              "12997215  743546441591721984      [1447]\n",
              "\n",
              "[4619330 rows x 2 columns]"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_final = train_text[~train_text.annotations.isnull()]\n",
        "train_final"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>746634523048292352</td>\n",
              "      <td>[1389]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>741650544847147009</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>746803092239196162</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>744724790485757952</td>\n",
              "      <td>[1620]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>747681608430387200</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999990</th>\n",
              "      <td>741822517090918401</td>\n",
              "      <td>[186]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999991</th>\n",
              "      <td>742212828476510209</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999995</th>\n",
              "      <td>747420757790162944</td>\n",
              "      <td>[1620]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999998</th>\n",
              "      <td>745039072628899841</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999999</th>\n",
              "      <td>746762117898895361</td>\n",
              "      <td>[186]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>354147 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        id annotations\n",
              "0       746634523048292352      [1389]\n",
              "1       741650544847147009      [1381]\n",
              "2       746803092239196162      [1424]\n",
              "4       744724790485757952      [1620]\n",
              "8       747681608430387200      [1424]\n",
              "...                    ...         ...\n",
              "999990  741822517090918401       [186]\n",
              "999991  742212828476510209      [1381]\n",
              "999995  747420757790162944      [1620]\n",
              "999998  745039072628899841      [1447]\n",
              "999999  746762117898895361       [186]\n",
              "\n",
              "[354147 rows x 2 columns]"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "valid_text[\"annotations\"] = valid_text[\"annotations\"].apply(label)\n",
        "valid_final = valid_text[~valid_text.annotations.isnull()]\n",
        "valid_final"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>747643690521341953</td>\n",
              "      <td>[1138]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744595559470800896</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>743285629576843264</td>\n",
              "      <td>[1424]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>746745350023675905</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>742177996266606593</td>\n",
              "      <td>[186]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999988</th>\n",
              "      <td>743469083082002432</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999991</th>\n",
              "      <td>744928341900275712</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999993</th>\n",
              "      <td>748002024017530880</td>\n",
              "      <td>[1389]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999998</th>\n",
              "      <td>747071768817471488</td>\n",
              "      <td>[1389]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999999</th>\n",
              "      <td>748393225166127105</td>\n",
              "      <td>[1447]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>355734 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        id annotations\n",
              "0       747643690521341953      [1138]\n",
              "2       744595559470800896      [1381]\n",
              "4       743285629576843264      [1424]\n",
              "6       746745350023675905      [1381]\n",
              "7       742177996266606593       [186]\n",
              "...                    ...         ...\n",
              "999988  743469083082002432      [1447]\n",
              "999991  744928341900275712      [1381]\n",
              "999993  748002024017530880      [1389]\n",
              "999998  747071768817471488      [1389]\n",
              "999999  748393225166127105      [1447]\n",
              "\n",
              "[355734 rows x 2 columns]"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_text[\"annotations\"] = test_text[\"annotations\"].apply(label)\n",
        "test_final = test_text[~test_text.annotations.isnull()]\n",
        "test_final"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Put your emoji number to filter the train, test, and valid for that emoji.\n",
        "\n",
        "If you are doing for the first emoji put `emoji_number = 1` -- Hari"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "emoji_number = 1 # modify to 1-12\n",
        "def check(annotations):\n",
        "    if emoji_list_final[emoji_number-1] in annotations:\n",
        "        return True\n",
        "    return False\n",
        "\n",
        "train_text_emoji = train_final[train_final[\"annotations\"].apply(check)]\n",
        "test_text_emoji = test_final[test_final[\"annotations\"].apply(check)]\n",
        "valid_text_emoji= valid_final[valid_final[\"annotations\"].apply(check)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train size for emoji 1: (1216731, 2)\n",
            "Test size for emoji 1: (93600, 2)\n",
            "Validation size for emoji 1: (93216, 2)\n"
          ]
        }
      ],
      "source": [
        "print(f\"Train size for emoji {emoji_number}: {train_text_emoji.shape}\")\n",
        "print(f\"Test size for emoji {emoji_number}: {test_text_emoji.shape}\")\n",
        "print(f\"Validation size for emoji {emoji_number}: {valid_text_emoji.shape}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>annotations</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>744595559470800896</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>746745350023675905</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>742734050788544512</td>\n",
              "      <td>[1381, 1620]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>743918650323787776</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>743550851080421376</td>\n",
              "      <td>[1381]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                    id   annotations\n",
              "2   744595559470800896        [1381]\n",
              "6   746745350023675905        [1381]\n",
              "13  742734050788544512  [1381, 1620]\n",
              "27  743918650323787776        [1381]\n",
              "51  743550851080421376        [1381]"
            ]
          },
          "execution_count": 35,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_text_emoji.head() # just an example"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E5VipfI4IJQz"
      },
      "source": [
        "## Get The Tweets and Remove Unavailable Tweets\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "apnkG8Q6T1xc"
      },
      "outputs": [],
      "source": [
        "# consumer key / API key = BFkhoyYPvdDg6vRTgHAYyAa9r\n",
        "# consumer key secret / API key secret = Hx5OQOgLxP2fsqLoUUGRdJycBhisYg1uac8vtSqdLyLijUkvcD\n",
        "# bearer token = AAAAAAAAAAAAAAAAAAAAAHdSUQEAAAAAzd%2B01SA5YHfDQmy3h7VX2OYyiC4%3Dd2eQEh8Z2nAQ9zWBMyKlyjOKwjwb7lwUJT4SEDTWcCFxz3xHmX\n",
        "# access token = 1443079270301978625-vzmeZhaly1A7mmS861eBoF9eAF2eWW\n",
        "# access token secret = MR3rzfGlS9MHsT3B7bDBfwU7yAU5RS7F8MVYtSoGXmHGj"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "Q6oY6swzYdTC"
      },
      "outputs": [],
      "source": [
        "import tweepy\n",
        "\n",
        "consumer_key = 'BFkhoyYPvdDg6vRTgHAYyAa9r'\n",
        "consumer_secret = 'Hx5OQOgLxP2fsqLoUUGRdJycBhisYg1uac8vtSqdLyLijUkvcD'\n",
        "access_key= '1443079270301978625-vzmeZhaly1A7mmS861eBoF9eAF2eWW'\n",
        "access_secret = 'MR3rzfGlS9MHsT3B7bDBfwU7yAU5RS7F8MVYtSoGXmHGj'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "M54pOYVgY5ci"
      },
      "outputs": [],
      "source": [
        "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
        "auth.set_access_token(access_key, access_secret)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "s7tsbm8HZwe9"
      },
      "outputs": [],
      "source": [
        "api = tweepy.API(auth)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "**WARNING**\n",
        "\n",
        "There might be a problem if you run the tests multiple times\n",
        "\n",
        "If you encounter problems such as extracting tweets too fast \n",
        "\n",
        "Or the dataset returns all empty tweets\n",
        "\n",
        "Just restart the notebook and run for that specific set\n",
        "\n",
        "-- Hari"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {},
      "outputs": [],
      "source": [
        "count_valid, count_test, count_train = 0, 0, 0\n",
        "def get_tweets_valid(id):\n",
        "    global count_valid\n",
        "    try:\n",
        "        if count_valid >= 2000:\n",
        "            return\n",
        "        x = api.get_status(id).text\n",
        "        count_valid += 1\n",
        "        if count_valid % 20 == 0:\n",
        "            print(f\"Checkpoint {count_valid}\")\n",
        "        return x\n",
        "    except:\n",
        "        return\n",
        "\n",
        "def get_tweets_test(id):\n",
        "    global count_test\n",
        "    try:\n",
        "        if count_test >= 2000:\n",
        "            return\n",
        "        x = api.get_status(id).text\n",
        "        count_test += 1\n",
        "        if count_test % 20 == 0:\n",
        "            print(f\"Checkpoint {count_test}\")\n",
        "        return x\n",
        "    except:\n",
        "        return\n",
        "\n",
        "def get_tweets_train(id):\n",
        "    global count_train\n",
        "    try:\n",
        "        if count_train >= 5000:\n",
        "            return\n",
        "        x = api.get_status(id).text\n",
        "        count_train += 1\n",
        "        if count_train % 20 == 0:\n",
        "            print(f\"Checkpoint {count_train}\")\n",
        "        return x\n",
        "    except:\n",
        "        return"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Sample 100000 random from the emoji sample set for each train, test, and valid"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [],
      "source": [
        "train_text_emoji = train_text_emoji.sample(n=100000, random_state=23, ignore_index=True, replace=True)\n",
        "test_text_emoji = test_text_emoji.sample(n=100000, random_state=23, ignore_index=True, replace=True)\n",
        "valid_text_emoji = valid_text_emoji.sample(n=100000, random_state=23, ignore_index=True, replace=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Get tweets for train, test, and validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "train_text_emoji[\"tweets\"] = train_text_emoji[\"id\"].apply(get_tweets_train)\n",
        "train_text_emoji = train_text_emoji[~test_text_emoji[\"tweets\"].isnull()]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "test_text_emoji[\"tweets\"] = test_text_emoji[\"id\"].apply(get_tweets_test)\n",
        "test_text_emoji = test_text_emoji[~test_text_emoji[\"tweets\"].isnull()]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "valid_text_emoji[\"tweets\"] = valid_text_emoji[\"id\"].apply(get_tweets_valid)\n",
        "valid_text_emoji = valid_text_emoji[~test_text_emoji[\"tweets\"].isnull()]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Saving final files to CSV"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "train_text_emoji.to_csv(dataset_path + 'train_text_emoji.csv', index=False)\n",
        "test_text_emoji.to_csv(dataset_path + 'test_text_emoji.csv', index=False)\n",
        "valid_text_emoji.to_csv(dataset_path + 'valid_text_emoji.csv', index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Combining all processes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def get_csv_from_emoji(emoji_number):\n",
        "    def check(annotations):\n",
        "        if emoji_list_final[emoji_number-1] in annotations:\n",
        "            return True\n",
        "        return False\n",
        "\n",
        "    train_text_emoji = train_final[train_final[\"annotations\"].apply(check)]\n",
        "    test_text_emoji = test_final[test_final[\"annotations\"].apply(check)]\n",
        "    valid_text_emoji = valid_final[valid_final[\"annotations\"].apply(check)]\n",
        "\n",
        "    print(f\"Train size for emoji {emoji_number}: {train_text_emoji.shape}\")\n",
        "    print(f\"Test size for emoji {emoji_number}: {test_text_emoji.shape}\")\n",
        "    print(f\"Validation size for emoji {emoji_number}: {valid_text_emoji.shape}\")\n",
        "\n",
        "    count_valid, count_test, count_train = 0, 0, 0\n",
        "    \n",
        "    def get_tweets_valid(id):\n",
        "        nonlocal count_valid\n",
        "        try:\n",
        "            if count_valid >= 2000:\n",
        "                return\n",
        "            x = api.get_status(id).text\n",
        "            count_valid += 1\n",
        "            if count_valid % 20 == 0:\n",
        "                print(f\"Checkpoint {count_valid}\")\n",
        "            return x\n",
        "        except:\n",
        "            return\n",
        "\n",
        "    def get_tweets_test(id):\n",
        "        nonlocal count_test\n",
        "        try:\n",
        "            if count_test >= 2000:\n",
        "                return\n",
        "            x = api.get_status(id).text\n",
        "            count_test += 1\n",
        "            if count_test % 20 == 0:\n",
        "                print(f\"Checkpoint {count_test}\")\n",
        "            return x\n",
        "        except:\n",
        "            return\n",
        "\n",
        "    def get_tweets_train(id):\n",
        "        nonlocal count_train\n",
        "        try:\n",
        "            if count_train >= 5000:\n",
        "                return\n",
        "            x = api.get_status(id).text\n",
        "            count_train += 1\n",
        "            if count_train % 20 == 0:\n",
        "                print(f\"Checkpoint {count_train}\")\n",
        "            return x\n",
        "        except:\n",
        "            return\n",
        "\n",
        "    train_text_emoji[\"tweets\"] = train_text_emoji[\"id\"].apply(get_tweets_train)\n",
        "    train_text_emoji = train_text_emoji[~test_text_emoji[\"tweets\"].isnull()]\n",
        "\n",
        "    test_text_emoji[\"tweets\"] = test_text_emoji[\"id\"].apply(get_tweets_test)\n",
        "    test_text_emoji = test_text_emoji[~test_text_emoji[\"tweets\"].isnull()]\n",
        "\n",
        "    valid_text_emoji[\"tweets\"] = valid_text_emoji[\"id\"].apply(get_tweets_valid)\n",
        "    valid_text_emoji = valid_text_emoji[~test_text_emoji[\"tweets\"].isnull()]\n",
        "\n",
        "    train_text_emoji.to_csv(dataset_path + f'train_text_emoji_{emoji_number}.csv', index=False)\n",
        "    test_text_emoji.to_csv(dataset_path + f'test_text_emoji_{emoji_number}.csv', index=False)\n",
        "    valid_text_emoji.to_csv(dataset_path + f'valid_text_emoji_{emoji_number}.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Will take a very long time!\n",
        "for num in range(1, 13):\n",
        "    get_csv_from_emoji(num)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "BLWqY7vn4f1T"
      ],
      "name": "CS3244 Twemoji.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "247ab06e135bb35fa78c5eff31b2a9a0050dcb5fb773c2631d2a29ac689eeccb"
    },
    "kernelspec": {
      "display_name": "Python 3.9.6 64-bit",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}