Pipeline-NLP-POS (Simple Transformer).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0b710f20-19b8-47cf-873e-88ecd5a95b83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Mar 24 23:03:06 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 470.103.01   Driver Version: 470.103.01   CUDA Version: 11.4     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA A100-SXM...  On   | 00000000:03:00.0 Off |                    0 |\n",
      "| N/A   33C    P0    50W / 400W |      0MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2cfdc6e9-117e-496c-9d72-bd5c0764eada",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from tqdm.notebook import tqdm\n",
    "import torch\n",
    "import numpy as np\n",
    "from simpletransformers.ner import NERModel, NERArgs\n",
    "import pandas as pd\n",
    "import datasets\n",
    "from datasets import load_metric\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ff34ba96-7513-402f-9018-20d4e225c326",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_txt = open(\"LST20/train/T00126.txt\", \"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fbeacabf-9061-4157-9c63-951984bcc789",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_txt = sample_txt.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7ce2623b-6a91-4609-b25b-eeda270b98ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_txt = sample_txt.split(\"\\n\")[0].split(\"\\t\") #concept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ac01d8e5-c83f-4e8f-87b6-8a48b2f685fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NN'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_txt[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "74d86c43-5cc3-45d1-a764-8a78e853b7f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "lst20_train_path = \"LST20/train/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "329f0a56-f326-418c-bca8-9641b3f271ac",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72b21261ac91439bb84f7872f6657378",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3795 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df = pd.DataFrame()\n",
    "name_list = []\n",
    "pos_list = []\n",
    "ner1_list = []\n",
    "ner2_list = []\n",
    "\n",
    "for file_text in tqdm(os.listdir(lst20_train_path)):\n",
    "    path = lst20_train_path + file_text\n",
    "    if path.endswith(\"txt\"):\n",
    "        sample_text = open(path)\n",
    "        sample_text = sample_text.read()\n",
    "        n_txt = len(sample_text.split(\"\\n\"))\n",
    "        for i in range(n_txt):\n",
    "            all_txt = sample_text.split(\"\\n\")[i].split(\"\\t\") #concept\n",
    "            if len(all_txt)>1 :\n",
    "                name_list.append(all_txt[0])\n",
    "                pos_list.append(all_txt[1])\n",
    "                ner1_list.append(all_txt[2])\n",
    "                ner2_list.append(all_txt[3])\n",
    "            else:\n",
    "                pass\n",
    "        else:\n",
    "            pass\n",
    "    # print(file_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "db12c3da-886a-4664-baaa-7e19fff4640f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df[\"name\"] = name_list\n",
    "train_df[\"pos\"] = pos_list\n",
    "train_df[\"ner1\"] = ner1_list\n",
    "train_df[\"ner2\"] = ner2_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "54b1a7d4-ff9c-438d-874c-ab388277c8ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>pos</th>\n",
       "      <th>ner1</th>\n",
       "      <th>ner2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ธ.กสิกรไทย</td>\n",
       "      <td>NN</td>\n",
       "      <td>B_ORG</td>\n",
       "      <td>B_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>จับ</td>\n",
       "      <td>VV</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>มือ</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>เวอริไซน์</td>\n",
       "      <td>NN</td>\n",
       "      <td>B_ORG</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>_</td>\n",
       "      <td>PU</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2714721</th>\n",
       "      <td>ตาม</td>\n",
       "      <td>PS</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2714722</th>\n",
       "      <td>ราคา</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2714723</th>\n",
       "      <td>ตลาด</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2714724</th>\n",
       "      <td>รวม</td>\n",
       "      <td>VV</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2714725</th>\n",
       "      <td>ทั้งหมด</td>\n",
       "      <td>AJ</td>\n",
       "      <td>O</td>\n",
       "      <td>E_CLS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2714726 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               name pos   ner1   ner2\n",
       "0        ธ.กสิกรไทย  NN  B_ORG  B_CLS\n",
       "1               จับ  VV      O  I_CLS\n",
       "2               มือ  NN      O  I_CLS\n",
       "3         เวอริไซน์  NN  B_ORG  I_CLS\n",
       "4                 _  PU      O  I_CLS\n",
       "...             ...  ..    ...    ...\n",
       "2714721         ตาม  PS      O  I_CLS\n",
       "2714722        ราคา  NN      O  I_CLS\n",
       "2714723        ตลาด  NN      O  I_CLS\n",
       "2714724         รวม  VV      O  I_CLS\n",
       "2714725     ทั้งหมด  AJ      O  E_CLS\n",
       "\n",
       "[2714726 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2c0cfd94-7844-4d3d-b6c0-5bc08b56fd48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b37d8276e8d4eb0b4b7d53de3bead76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/475 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lst20_eval_path = \"LST20/eval/\"\n",
    "\n",
    "eval_df = pd.DataFrame()\n",
    "eval_name_list = []\n",
    "eval_pos_list = []\n",
    "eval_ner1_list = []\n",
    "eval_ner2_list = []\n",
    "\n",
    "for file_text in tqdm(os.listdir(lst20_eval_path)):\n",
    "    path = lst20_eval_path + file_text\n",
    "    if path.endswith(\"txt\"):\n",
    "        sample_text = open(path)\n",
    "        sample_text = sample_text.read()\n",
    "        n_txt = len(sample_text.split(\"\\n\"))\n",
    "        for i in range(n_txt):\n",
    "            all_txt = sample_text.split(\"\\n\")[i].split(\"\\t\") #concept\n",
    "            if len(all_txt)>1 :\n",
    "                eval_name_list.append(all_txt[0])\n",
    "                eval_pos_list.append(all_txt[1])\n",
    "                eval_ner1_list.append(all_txt[2])\n",
    "                eval_ner2_list.append(all_txt[3])\n",
    "            else:\n",
    "                pass\n",
    "        else:\n",
    "            pass\n",
    "    # print(file_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3dd00535-6138-4743-82b9-7c9346c8dbe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df[\"name\"] = eval_name_list\n",
    "eval_df[\"pos\"] = eval_pos_list\n",
    "eval_df[\"ner1\"] = eval_ner1_list\n",
    "eval_df[\"ner2\"] = eval_ner2_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9999bfae-3268-4da7-afb7-31e2a2fa5445",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>pos</th>\n",
       "      <th>ner1</th>\n",
       "      <th>ner2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>นายก</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "      <td>B_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ฯ</td>\n",
       "      <td>PU</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>_</td>\n",
       "      <td>PU</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ย้ำ</td>\n",
       "      <td>VV</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>_</td>\n",
       "      <td>PU</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240855</th>\n",
       "      <td>ตำแหน่ง</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240856</th>\n",
       "      <td>หัวหน้า</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240857</th>\n",
       "      <td>พรรค</td>\n",
       "      <td>NN</td>\n",
       "      <td>B_ORG</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240858</th>\n",
       "      <td>เพื่อ</td>\n",
       "      <td>PS</td>\n",
       "      <td>I_ORG</td>\n",
       "      <td>I_CLS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240859</th>\n",
       "      <td>ไทย</td>\n",
       "      <td>NN</td>\n",
       "      <td>E_ORG</td>\n",
       "      <td>E_CLS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240860 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           name pos   ner1   ner2\n",
       "0          นายก  NN      O  B_CLS\n",
       "1             ฯ  PU      O  I_CLS\n",
       "2             _  PU      O  I_CLS\n",
       "3           ย้ำ  VV      O  I_CLS\n",
       "4             _  PU      O  I_CLS\n",
       "...         ...  ..    ...    ...\n",
       "240855  ตำแหน่ง  NN      O  I_CLS\n",
       "240856  หัวหน้า  NN      O  I_CLS\n",
       "240857     พรรค  NN  B_ORG  I_CLS\n",
       "240858    เพื่อ  PS  I_ORG  I_CLS\n",
       "240859      ไทย  NN  E_ORG  E_CLS\n",
       "\n",
       "[240860 rows x 4 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "86655549-4b8b-4ed7-a2c2-37250e413965",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_df = train_df[[\"name\",\"pos\"]]\n",
    "new_eval_df = eval_df[[\"name\", \"pos\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3fbfcf7c-f85e-4798-89ea-5a51e6a16468",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_df = new_train_df.rename(columns={\"name\":\"text\", \"pos\":\"label\"})\n",
    "new_eval_df = new_eval_df.rename(columns={\"name\":\"text\", \"pos\":\"label\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1b10c789-b7f6-44ef-a9ef-2d2260e2e273",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: >"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGkCAYAAADdUkoCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7yUlEQVR4nO3de1xVdb7/8Tcb5OJlg5hAHPEyo6akaYLi7joWuTPs5IiNmsdI0cqDlnLGW8ewbEbNJm+pUZNKTTmpUzkpiREetUbygjmpjY6WhkUbbQp2OgoK6/fH/FjDVhRQNqjr9Xw81uPhXt/PXp+vG/bizdprLXwMwzAEAABgQbaGngAAAEBDIQgBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADL8mvoCVzJysvLVVBQoGbNmsnHx6ehpwMAAGrAMAz99NNPioyMlM128WM+BKGLKCgoUFRUVENPAwAAXIKjR4+qVatWF60hCF1Es2bNJP3rhbTb7Q08GwAAUBNut1tRUVHmz/GLIQhdRMXHYXa7nSAEAMBVpiantXCyNAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCy/hp7A1a7tlMzLev6R2Ql1NBMAAFBbHBECAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWVasg1LZtW/n4+Jy3pKSkSJJOnz6tlJQUtWjRQk2bNlViYqIKCws9tpGfn6+EhAQ1btxYYWFhmjhxos6ePetRs2nTJvXo0UMBAQFq3769MjIyzpvL4sWL1bZtWwUGBiouLk7bt2/3GK/JXAAAgLXVKgjt2LFD3333nblkZ2dLkh588EFJ0oQJE7R27VqtXr1amzdvVkFBgQYOHGg+v6ysTAkJCSotLdXWrVv1+uuvKyMjQ2lpaWbN4cOHlZCQoD59+mj37t0aP368Ro0apQ0bNpg1K1euVGpqqqZPn65du3apW7ducjqdOnbsmFlT3VwAAAB8DMMwLvXJ48eP17p163Tw4EG53W61bNlSK1as0KBBgyRJ+/fvV+fOnZWbm6vevXtr/fr16t+/vwoKChQeHi5JSk9P1+TJk3X8+HH5+/tr8uTJyszM1N69e80+Q4YMUVFRkbKysiRJcXFx6tmzpxYtWiRJKi8vV1RUlMaNG6cpU6aouLi42rnUhNvtVnBwsIqLi2W326usaTsl89JevP/vyOyEy3o+AADwVJOf3xUu+Ryh0tJSvfnmmxo5cqR8fHyUl5enM2fOKD4+3qzp1KmTWrdurdzcXElSbm6uunbtaoYgSXI6nXK73dq3b59ZU3kbFTUV2ygtLVVeXp5Hjc1mU3x8vFlTk7lUpaSkRG6322MBAADXrksOQmvWrFFRUZEeeeQRSZLL5ZK/v79CQkI86sLDw+VyucyayiGoYrxi7GI1brdbp06d0vfff6+ysrIqaypvo7q5VGXWrFkKDg42l6ioqOpfCAAAcNW65CC0dOlS9evXT5GRkXU5nwY1depUFRcXm8vRo0cbekoAAMCL/C7lSV9//bU++ugjvfvuu+a6iIgIlZaWqqioyONITGFhoSIiIsyac6/uqriSq3LNuVd3FRYWym63KygoSL6+vvL19a2ypvI2qptLVQICAhQQEFDDVwEAAFztLumI0PLlyxUWFqaEhH+f6BsTE6NGjRopJyfHXHfgwAHl5+fL4XBIkhwOh/bs2eNxdVd2drbsdruio6PNmsrbqKip2Ia/v79iYmI8asrLy5WTk2PW1GQuAAAAtT4iVF5eruXLlyspKUl+fv9+enBwsJKTk5WamqrQ0FDZ7XaNGzdODofDvEqrb9++io6O1vDhwzVnzhy5XC5NmzZNKSkp5pGYxx9/XIsWLdKkSZM0cuRIbdy4UatWrVJm5r+vzkpNTVVSUpJiY2PVq1cvzZ8/XydPntSIESNqPBcAAIBaB6GPPvpI+fn5Gjly5Hlj8+bNk81mU2JiokpKSuR0OrVkyRJz3NfXV+vWrdOYMWPkcDjUpEkTJSUlacaMGWZNu3btlJmZqQkTJmjBggVq1aqVXnvtNTmdTrNm8ODBOn78uNLS0uRyudS9e3dlZWV5nEBd3VwAAAAu6z5C1zruIwQAwNWnXu4jBAAAcLUjCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMuqdRD69ttv9V//9V9q0aKFgoKC1LVrV+3cudMcNwxDaWlpuv766xUUFKT4+HgdPHjQYxs//PCDhg0bJrvdrpCQECUnJ+vEiRMeNZ9//rluv/12BQYGKioqSnPmzDlvLqtXr1anTp0UGBiorl276oMPPvAYr8lcAACAddUqCP3444+69dZb1ahRI61fv15ffPGFXnzxRTVv3tysmTNnjhYuXKj09HRt27ZNTZo0kdPp1OnTp82aYcOGad++fcrOzta6deu0ZcsWPfroo+a42+1W37591aZNG+Xl5emFF17QM888o1dffdWs2bp1q4YOHark5GR99tlnGjBggAYMGKC9e/fWai4AAMC6fAzDMGpaPGXKFP3lL3/Rxx9/XOW4YRiKjIzU//zP/+jXv/61JKm4uFjh4eHKyMjQkCFD9Le//U3R0dHasWOHYmNjJUlZWVm677779M033ygyMlIvv/yy/vd//1cul0v+/v5m7zVr1mj//v2SpMGDB+vkyZNat26d2b93797q3r270tPTazSX6rjdbgUHB6u4uFh2u73KmrZTMmv46lXtyOyEy3o+AADwVJOf3xVqdUTo/fffV2xsrB588EGFhYXp5ptv1u9//3tz/PDhw3K5XIqPjzfXBQcHKy4uTrm5uZKk3NxchYSEmCFIkuLj42Wz2bRt2zaz5o477jBDkCQ5nU4dOHBAP/74o1lTuU9FTUWfmswFAABYW62C0FdffaWXX35ZHTp00IYNGzRmzBg98cQTev311yVJLpdLkhQeHu7xvPDwcHPM5XIpLCzMY9zPz0+hoaEeNVVto3KPC9VUHq9uLucqKSmR2+32WAAAwLXLrzbF5eXlio2N1cyZMyVJN998s/bu3av09HQlJSV5ZYL1adasWXr22WcbehoAAKCe1OqI0PXXX6/o6GiPdZ07d1Z+fr4kKSIiQpJUWFjoUVNYWGiORURE6NixYx7jZ8+e1Q8//OBRU9U2Kve4UE3l8ermcq6pU6equLjYXI4ePVplHQAAuDbUKgjdeuutOnDggMe6v//972rTpo0kqV27doqIiFBOTo457na7tW3bNjkcDkmSw+FQUVGR8vLyzJqNGzeqvLxccXFxZs2WLVt05swZsyY7O1s33HCDeYWaw+Hw6FNRU9GnJnM5V0BAgOx2u8cCAACuXbUKQhMmTNCnn36qmTNn6tChQ1qxYoVeffVVpaSkSJJ8fHw0fvx4/eY3v9H777+vPXv26OGHH1ZkZKQGDBgg6V9HkO69916NHj1a27dv11/+8heNHTtWQ4YMUWRkpCTpoYcekr+/v5KTk7Vv3z6tXLlSCxYsUGpqqjmXJ598UllZWXrxxRe1f/9+PfPMM9q5c6fGjh1b47kAAABrq9U5Qj179tR7772nqVOnasaMGWrXrp3mz5+vYcOGmTWTJk3SyZMn9eijj6qoqEi33XabsrKyFBgYaNa89dZbGjt2rO6++27ZbDYlJiZq4cKF5nhwcLA+/PBDpaSkKCYmRtddd53S0tI87jV0yy23aMWKFZo2bZqeeuopdejQQWvWrFGXLl1qNRcAAGBdtbqPkNVwHyEAAK4+XruPEAAAwLWEIAQAACyLIAQAACyLIAQAACyLIAQAACyrVpfP48rCFWsAAFwejggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLIggBAADLqlUQeuaZZ+Tj4+OxdOrUyRw/ffq0UlJS1KJFCzVt2lSJiYkqLCz02EZ+fr4SEhLUuHFjhYWFaeLEiTp79qxHzaZNm9SjRw8FBASoffv2ysjIOG8uixcvVtu2bRUYGKi4uDht377dY7wmcwEAANZW6yNCN954o7777jtz+eSTT8yxCRMmaO3atVq9erU2b96sgoICDRw40BwvKytTQkKCSktLtXXrVr3++uvKyMhQWlqaWXP48GElJCSoT58+2r17t8aPH69Ro0Zpw4YNZs3KlSuVmpqq6dOna9euXerWrZucTqeOHTtW47kAAAD4GIZh1LT4mWee0Zo1a7R79+7zxoqLi9WyZUutWLFCgwYNkiTt379fnTt3Vm5urnr37q3169erf//+KigoUHh4uCQpPT1dkydP1vHjx+Xv76/JkycrMzNTe/fuNbc9ZMgQFRUVKSsrS5IUFxennj17atGiRZKk8vJyRUVFady4cZoyZUqN5lITbrdbwcHBKi4ult1ur7Km7ZTMmr14F3BkdsIlP7chewMAcKWqyc/vCrU+InTw4EFFRkbqZz/7mYYNG6b8/HxJUl5ens6cOaP4+HiztlOnTmrdurVyc3MlSbm5ueratasZgiTJ6XTK7XZr3759Zk3lbVTUVGyjtLRUeXl5HjU2m03x8fFmTU3mUpWSkhK53W6PBQAAXLtqFYTi4uKUkZGhrKwsvfzyyzp8+LBuv/12/fTTT3K5XPL391dISIjHc8LDw+VyuSRJLpfLIwRVjFeMXazG7Xbr1KlT+v7771VWVlZlTeVtVDeXqsyaNUvBwcHmEhUVVbMXBgAAXJX8alPcr18/89833XST4uLi1KZNG61atUpBQUF1Prn6NnXqVKWmppqP3W43YQgAgGvYZV0+HxISoo4dO+rQoUOKiIhQaWmpioqKPGoKCwsVEREhSYqIiDjvyq2Kx9XV2O12BQUF6brrrpOvr2+VNZW3Ud1cqhIQECC73e6xAACAa9dlBaETJ07oyy+/1PXXX6+YmBg1atRIOTk55viBAweUn58vh8MhSXI4HNqzZ4/H1V3Z2dmy2+2Kjo42aypvo6KmYhv+/v6KiYnxqCkvL1dOTo5ZU5O5AAAA1OqjsV//+te6//771aZNGxUUFGj69Ony9fXV0KFDFRwcrOTkZKWmpio0NFR2u13jxo2Tw+Ewr9Lq27evoqOjNXz4cM2ZM0cul0vTpk1TSkqKAgICJEmPP/64Fi1apEmTJmnkyJHauHGjVq1apczMf18hlZqaqqSkJMXGxqpXr16aP3++Tp48qREjRkhSjeYCAABQqyD0zTffaOjQofrHP/6hli1b6rbbbtOnn36qli1bSpLmzZsnm82mxMRElZSUyOl0asmSJebzfX19tW7dOo0ZM0YOh0NNmjRRUlKSZsyYYda0a9dOmZmZmjBhghYsWKBWrVrptddek9PpNGsGDx6s48ePKy0tTS6XS927d1dWVpbHCdTVzQUAAKBW9xGyGu4jBADA1cer9xECAAC4VhCEAACAZRGEAACAZRGEAACAZRGEAACAZRGEAACAZRGEAACAZRGEAACAZRGEAACAZRGEAACAZRGEAACAZRGEAACAZdXqr88DFfiDrwCAawFHhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGURhAAAgGVdVhCaPXu2fHx8NH78eHPd6dOnlZKSohYtWqhp06ZKTExUYWGhx/Py8/OVkJCgxo0bKywsTBMnTtTZs2c9ajZt2qQePXooICBA7du3V0ZGxnn9Fy9erLZt2yowMFBxcXHavn27x3hN5gIAAKzrkoPQjh079Morr+imm27yWD9hwgStXbtWq1ev1ubNm1VQUKCBAwea42VlZUpISFBpaam2bt2q119/XRkZGUpLSzNrDh8+rISEBPXp00e7d+/W+PHjNWrUKG3YsMGsWblypVJTUzV9+nTt2rVL3bp1k9Pp1LFjx2o8FwAAYG2XFIROnDihYcOG6fe//72aN29uri8uLtbSpUs1d+5c3XXXXYqJidHy5cu1detWffrpp5KkDz/8UF988YXefPNNde/eXf369dNzzz2nxYsXq7S0VJKUnp6udu3a6cUXX1Tnzp01duxYDRo0SPPmzTN7zZ07V6NHj9aIESMUHR2t9PR0NW7cWMuWLavxXAAAgLVdUhBKSUlRQkKC4uPjPdbn5eXpzJkzHus7deqk1q1bKzc3V5KUm5urrl27Kjw83KxxOp1yu93at2+fWXPutp1Op7mN0tJS5eXledTYbDbFx8ebNTWZy7lKSkrkdrs9FgAAcO3yq+0T3n77be3atUs7duw4b8zlcsnf318hISEe68PDw+VyucyayiGoYrxi7GI1brdbp06d0o8//qiysrIqa/bv31/juZxr1qxZevbZZy/yvwcAANeSWh0ROnr0qJ588km99dZbCgwM9NacGszUqVNVXFxsLkePHm3oKQEAAC+qVRDKy8vTsWPH1KNHD/n5+cnPz0+bN2/WwoUL5efnp/DwcJWWlqqoqMjjeYWFhYqIiJAkRUREnHflVsXj6mrsdruCgoJ03XXXydfXt8qaytuobi7nCggIkN1u91gAAMC1q1ZB6O6779aePXu0e/duc4mNjdWwYcPMfzdq1Eg5OTnmcw4cOKD8/Hw5HA5JksPh0J49ezyu7srOzpbdbld0dLRZU3kbFTUV2/D391dMTIxHTXl5uXJycsyamJiYaucCAACsrVbnCDVr1kxdunTxWNekSRO1aNHCXJ+cnKzU1FSFhobKbrdr3Lhxcjgc6t27tySpb9++io6O1vDhwzVnzhy5XC5NmzZNKSkpCggIkCQ9/vjjWrRokSZNmqSRI0dq48aNWrVqlTIzM82+qampSkpKUmxsrHr16qX58+fr5MmTGjFihCQpODi42rkAAABrq/XJ0tWZN2+ebDabEhMTVVJSIqfTqSVLlpjjvr6+WrduncaMGSOHw6EmTZooKSlJM2bMMGvatWunzMxMTZgwQQsWLFCrVq302muvyel0mjWDBw/W8ePHlZaWJpfLpe7duysrK8vjBOrq5gIAAKzNxzAMo6EncaVyu90KDg5WcXHxBc8Xajsls8r1NXVkdsIlP9eqvQEAuJia/PyuwN8aAwAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAlkUQAgAAllWrIPTyyy/rpptukt1ul91ul8Ph0Pr1683x06dPKyUlRS1atFDTpk2VmJiowsJCj23k5+crISFBjRs3VlhYmCZOnKizZ8961GzatEk9evRQQECA2rdvr4yMjPPmsnjxYrVt21aBgYGKi4vT9u3bPcZrMhcAAGBttQpCrVq10uzZs5WXl6edO3fqrrvu0gMPPKB9+/ZJkiZMmKC1a9dq9erV2rx5swoKCjRw4EDz+WVlZUpISFBpaam2bt2q119/XRkZGUpLSzNrDh8+rISEBPXp00e7d+/W+PHjNWrUKG3YsMGsWblypVJTUzV9+nTt2rVL3bp1k9Pp1LFjx8ya6uYCAADgYxiGcTkbCA0N1QsvvKBBgwapZcuWWrFihQYNGiRJ2r9/vzp37qzc3Fz17t1b69evV//+/VVQUKDw8HBJUnp6uiZPnqzjx4/L399fkydPVmZmpvbu3Wv2GDJkiIqKipSVlSVJiouLU8+ePbVo0SJJUnl5uaKiojRu3DhNmTJFxcXF1c6lJtxut4KDg1VcXCy73V5lTdspmZf2wv1/R2YnXPJzrdobAICLqcnP7wqXfI5QWVmZ3n77bZ08eVIOh0N5eXk6c+aM4uPjzZpOnTqpdevWys3NlSTl5uaqa9euZgiSJKfTKbfbbR5Vys3N9dhGRU3FNkpLS5WXl+dRY7PZFB8fb9bUZC5VKSkpkdvt9lgAAMC1q9ZBaM+ePWratKkCAgL0+OOP67333lN0dLRcLpf8/f0VEhLiUR8eHi6XyyVJcrlcHiGoYrxi7GI1brdbp06d0vfff6+ysrIqaypvo7q5VGXWrFkKDg42l6ioqJq9KAAA4KpU6yB0ww03aPfu3dq2bZvGjBmjpKQkffHFF96YW72bOnWqiouLzeXo0aMNPSUAAOBFfrV9gr+/v9q3by9JiomJ0Y4dO7RgwQINHjxYpaWlKioq8jgSU1hYqIiICElSRETEeVd3VVzJVbnm3Ku7CgsLZbfbFRQUJF9fX/n6+lZZU3kb1c2lKgEBAQoICKjFqwEAAK5ml30fofLycpWUlCgmJkaNGjVSTk6OOXbgwAHl5+fL4XBIkhwOh/bs2eNxdVd2drbsdruio6PNmsrbqKip2Ia/v79iYmI8asrLy5WTk2PW1GQuAAAAtToiNHXqVPXr10+tW7fWTz/9pBUrVmjTpk3asGGDgoODlZycrNTUVIWGhsput2vcuHFyOBzmVVp9+/ZVdHS0hg8frjlz5sjlcmnatGlKSUkxj8Q8/vjjWrRokSZNmqSRI0dq48aNWrVqlTIz/32VUmpqqpKSkhQbG6tevXpp/vz5OnnypEaMGCFJNZoLAABArYLQsWPH9PDDD+u7775TcHCwbrrpJm3YsEH33HOPJGnevHmy2WxKTExUSUmJnE6nlixZYj7f19dX69at05gxY+RwONSkSRMlJSVpxowZZk27du2UmZmpCRMmaMGCBWrVqpVee+01OZ1Os2bw4ME6fvy40tLS5HK51L17d2VlZXmcQF3dXAAAAC77PkLXMu4jdGX2BgDgYurlPkIAAABXO4IQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLIIQAACwLL+GngBQW22nZF7W84/MTqijmQAArnYcEQIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZFEAIAAJZVqyA0a9Ys9ezZU82aNVNYWJgGDBigAwcOeNScPn1aKSkpatGihZo2barExEQVFhZ61OTn5yshIUGNGzdWWFiYJk6cqLNnz3rUbNq0ST169FBAQIDat2+vjIyM8+azePFitW3bVoGBgYqLi9P27dtrPRcAAGBdtQpCmzdvVkpKij799FNlZ2frzJkz6tu3r06ePGnWTJgwQWvXrtXq1au1efNmFRQUaODAgeZ4WVmZEhISVFpaqq1bt+r1119XRkaG0tLSzJrDhw8rISFBffr00e7duzV+/HiNGjVKGzZsMGtWrlyp1NRUTZ8+Xbt27VK3bt3kdDp17NixGs8FAABYm49hGMalPvn48eMKCwvT5s2bdccdd6i4uFgtW7bUihUrNGjQIEnS/v371blzZ+Xm5qp3795av369+vfvr4KCAoWHh0uS0tPTNXnyZB0/flz+/v6aPHmyMjMztXfvXrPXkCFDVFRUpKysLElSXFycevbsqUWLFkmSysvLFRUVpXHjxmnKlCk1mkt13G63goODVVxcLLvdXmVN2ymZl/rySZKOzE645OfSu/57AwCufDX5+V3hss4RKi4uliSFhoZKkvLy8nTmzBnFx8ebNZ06dVLr1q2Vm5srScrNzVXXrl3NECRJTqdTbrdb+/btM2sqb6OipmIbpaWlysvL86ix2WyKj483a2oyl3OVlJTI7XZ7LAAA4Np1yUGovLxc48eP16233qouXbpIklwul/z9/RUSEuJRGx4eLpfLZdZUDkEV4xVjF6txu906deqUvv/+e5WVlVVZU3kb1c3lXLNmzVJwcLC5REVF1fDVAAAAV6NLDkIpKSnau3ev3n777bqcT4OaOnWqiouLzeXo0aMNPSUAAOBFfpfypLFjx2rdunXasmWLWrVqZa6PiIhQaWmpioqKPI7EFBYWKiIiwqw59+quiiu5Ktece3VXYWGh7Ha7goKC5OvrK19f3yprKm+jurmcKyAgQAEBAbV4JQAAwNWsVkeEDMPQ2LFj9d5772njxo1q166dx3hMTIwaNWqknJwcc92BAweUn58vh8MhSXI4HNqzZ4/H1V3Z2dmy2+2Kjo42aypvo6KmYhv+/v6KiYnxqCkvL1dOTo5ZU5O5AAAAa6vVEaGUlBStWLFCf/7zn9WsWTPzXJvg4GAFBQUpODhYycnJSk1NVWhoqOx2u8aNGyeHw2FepdW3b19FR0dr+PDhmjNnjlwul6ZNm6aUlBTzaMzjjz+uRYsWadKkSRo5cqQ2btyoVatWKTPz31cLpaamKikpSbGxserVq5fmz5+vkydPasSIEeacqpsLAACwtloFoZdfflmS9Itf/MJj/fLly/XII49IkubNmyebzabExESVlJTI6XRqyZIlZq2vr6/WrVunMWPGyOFwqEmTJkpKStKMGTPMmnbt2ikzM1MTJkzQggUL1KpVK7322mtyOp1mzeDBg3X8+HGlpaXJ5XKpe/fuysrK8jiBurq5AAAAa7us+whd67iPEL0BAFeferuPEAAAwNWMIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACyLIAQAACzLr6EnAFxN2k7JvKznH5mdUEczAQDUBY4IAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAyyIIAQAAy6p1ENqyZYvuv/9+RUZGysfHR2vWrPEYNwxDaWlpuv766xUUFKT4+HgdPHjQo+aHH37QsGHDZLfbFRISouTkZJ04ccKj5vPPP9ftt9+uwMBARUVFac6cOefNZfXq1erUqZMCAwPVtWtXffDBB7WeC3C1aDsl87IWAMD5ah2ETp48qW7dumnx4sVVjs+ZM0cLFy5Uenq6tm3bpiZNmsjpdOr06dNmzbBhw7Rv3z5lZ2dr3bp12rJlix599FFz3O12q2/fvmrTpo3y8vL0wgsv6JlnntGrr75q1mzdulVDhw5VcnKyPvvsMw0YMEADBgzQ3r17azUXAABgXX61fUK/fv3Ur1+/KscMw9D8+fM1bdo0PfDAA5KkN954Q+Hh4VqzZo2GDBmiv/3tb8rKytKOHTsUGxsrSXrppZd033336Xe/+50iIyP11ltvqbS0VMuWLZO/v79uvPFG7d69W3PnzjUD04IFC3Tvvfdq4sSJkqTnnntO2dnZWrRokdLT02s0FwAAYG11eo7Q4cOH5XK5FB8fb64LDg5WXFyccnNzJUm5ubkKCQkxQ5AkxcfHy2azadu2bWbNHXfcIX9/f7PG6XTqwIED+vHHH82ayn0qair61GQu5yopKZHb7fZYAADAtatOg5DL5ZIkhYeHe6wPDw83x1wul8LCwjzG/fz8FBoa6lFT1TYq97hQTeXx6uZyrlmzZik4ONhcoqKiavC/BgAAVyuuGqtk6tSpKi4uNpejR4829JQAAIAX1WkQioiIkCQVFhZ6rC8sLDTHIiIidOzYMY/xs2fP6ocffvCoqWoblXtcqKbyeHVzOVdAQIDsdrvHAgAArl11GoTatWuniIgI5eTkmOvcbre2bdsmh8MhSXI4HCoqKlJeXp5Zs3HjRpWXlysuLs6s2bJli86cOWPWZGdn64YbblDz5s3Nmsp9Kmoq+tRkLgAAwNpqfdXYiRMndOjQIfPx4cOHtXv3boWGhqp169YaP368fvOb36hDhw5q166dnn76aUVGRmrAgAGSpM6dO+vee+/V6NGjlZ6erjNnzmjs2LEaMmSIIiMjJUkPPfSQnn32WSUnJ2vy5Mnau3evFixYoHnz5pl9n3zySd1555168cUXlZCQoLfffls7d+40L7H38fGpdi4AauZy70N0ZHZCHc0EAOpWrYPQzp071adPH/NxamqqJCkpKUkZGRmaNGmSTp48qUcffVRFRUW67bbblJWVpcDAQPM5b731lsaOHau7775bNptNiYmJWrhwoTkeHBysDz/8UCkpKYqJidF1112ntLQ0j3sN3XLLLVqxYoWmTZump556Sh06dNCaNWvUpUsXs6YmcwEAANZV6yD0i1/8QoZhXHDcx8dHM2bM0IwZMy5YExoaqhUrVly0z0033aSPP/74ojUPPvigHnzwwcuaC4ArG0ejAHgTV40BAADLIggBAADLqvVHYwBgFXwsB1z7OCIEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiz+xAQBXIP68B1A/OCIEAAAsiyAEAAAsi4/GAAAe+FgOVsIRIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFncRwgAcMXgHkaobxwRAgAAlsURIQAAxNEoq+KIEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCyCEAAAsCxLBKHFixerbdu2CgwMVFxcnLZv397QUwIAAFeAa/7O0itXrlRqaqrS09MVFxen+fPny+l06sCBAwoLC2vo6QEAwF2tG9A1f0Ro7ty5Gj16tEaMGKHo6Gilp6ercePGWrZsWUNPDQAANLBr+ohQaWmp8vLyNHXqVHOdzWZTfHy8cnNzz6svKSlRSUmJ+bi4uFiS5Ha7L9ijvOSflzXHi227OvSmN73pTW96X27va1HF62EYRvXFxjXs22+/NSQZW7du9Vg/ceJEo1evXufVT58+3ZDEwsLCwsLCcg0sR48erTYrXNNHhGpr6tSpSk1NNR+Xl5frhx9+UIsWLeTj41Pr7bndbkVFReno0aOy2+11OVV605ve9KY3vel9AYZh6KefflJkZGS1tdd0ELruuuvk6+urwsJCj/WFhYWKiIg4rz4gIEABAQEe60JCQi57Hna7vd6/gehNb3rTm970tnLv4ODgGtVd0ydL+/v7KyYmRjk5Oea68vJy5eTkyOFwNODMAADAleCaPiIkSampqUpKSlJsbKx69eql+fPn6+TJkxoxYkRDTw0AADSwaz4IDR48WMePH1daWppcLpe6d++urKwshYeHe713QECApk+fft7HbfWB3vSmN73pTW96V8/HMGpybRkAAMC155o+RwgAAOBiCEIAAMCyCEIAAMCyCEIAAMCyCEIAAMCyCEJXuUGDBikrK6tmf1gOV71jx45VW/Pxxx97pXdsbKzS09P5444wFRUVacWKFQ09DdSBhty3NDSC0FXuxx9/VEJCglq3bq20tDR99dVXDT2lenPw4EENHTq0yh/MxcXFeuihh+r99di8ebM++OAD/fjjj17ZfpcuXfSnP/2pyrFTp07piSee0N133+2V3t26ddOkSZN0/fXXa/jw4dq0aZNX+uDfPv/882qXL774Qj/88EODzO/rr7/W8OHDG6S3pAu+F65VhmFo/fr1GjRoUJ1vuyH3LQ2N+wjVkTfeeKNGdQ8//HCd9/7666+1fPlyvfHGG/r666915513atSoUUpMTPT6jajef//9KtcHBwerY8eOuv76673W+9FHH1VISIjmzJlT5fjkyZPldrv18ssv13nv559/XidOnNBzzz0n6V87qH79+unDDz+UJIWFhSknJ0c33nhjnfZ98cUX9fTTT+uBBx7QkiVL1Lx5c0n/+k1txIgRstlsWr58uW699dY67Vvhn//8p1atWqWMjAx9/PHHateunUaOHKmkpCT9x3/8h1d6Vnj66ac1ffp0+flVfR/Y/Px8JScnKzs7u857N9T3uc1mk4+PT7VHfH18fNStWze98cYb6tKli1fmUpW//vWv6tGjh8rKyryy/bNnz2r//v3y9/dXx44dzfV//vOflZaWpv3796ukpKTO+3755Zf67W9/q2XLlkmSWrdurRMnTpjjvr6++uSTT3TDDTfUee+qHD58WMuWLVNGRoaOHz+u+Ph4rVu3rk57NPS+ZeDAgdXW+Pn5KSIiQvfcc4/uv//+umte7d+nR42EhIRccGnevLnh7+9v2Gw2r88jJyfHGDZsmNG4cWOjefPmxn//938bO3fu9Fo/Hx+fCy42m8146KGHjJMnT3qld8eOHY3t27dfcHznzp1Gx44dvdL75ptvNt5++23z8apVq4ygoCDjk08+Mf7xj38YCQkJxoMPPuiV3vv27TNiY2ON66+/3li9erXxxBNPGH5+fsa4ceOMf/7zn17pWZVDhw4Z//u//2u0bt3a8PPzM+677z7jnXfe8Vq/qKgoo3v37saePXvOG0tPTzeaNWtm3HvvvV7p3VDf50eOHKl2+fzzz43c3Fxj4MCBxm233Vbnc7iY3bt3e22/tmfPHqNNmzaGzWYzbDab8ctf/tJwuVzGHXfcYYSGhhqTJ082jh496pXeTz75pDFlyhTzcdOmTY05c+YYGRkZRkZGhtGvXz/jscce80rvCqdPnzbefPNNo0+fPkajRo0Mm81mzJ071yguLvZaz4bctyQlJRmPPPLIRZeHH37YuPfee42goCDj6aefrrPeBCEvKygoMB577DGjUaNGhtPprLe+brfbSE9PN0JDQw1fX99661uhqKjIyMnJMTp16mRMnTrVKz0CAwONI0eOXHD8yJEjRlBQkFd6h4SEGF988YX5+JFHHjGGDx9uPs7NzTVatWrlld6GYRhnz541Bg8ebNhsNqNp06bGpk2bvNarOuXl5cbq1auN0NBQr4b94uJiY/jw4UZAQIAxc+ZMo6yszPj666+Nu+++27Db7cYrr7zitd4XUh/f51Vxu93GK6+8YvTq1ct8zQ8ePGg0bty43uZgGN4NQvfdd59x9913G2vXrjUeeughw8fHx+jUqZPxwgsveP2HcpcuXYxt27aZj5s2bWp8+eWX5uNNmzYZ7du390rvnTt3GmPGjDFCQkKM2NhYY8GCBYbL5TL8/PyMffv2eaVnZVfSvuVC1q5da0RFRdXZ9q75vzXWUH766Sc9//zzWrBggW688UZt2LBBffr0qZfehw8fVkZGhjIyMlRcXKz4+Ph66VtZcHCw7rrrLs2bN0/jx4/XzJkzvdLjyy+/VJs2baocP3TokOx2e533lf51yL7yx465ubkaP368+TgyMlLff/+9V3qfOXNG06dP17vvvqvBgwcrKytLM2fO1M9//nO1atXKKz0vZNOmTVq+fLneeecd+fn5afTo0V7rZbfb9cYbbygxMVGPPfaYVq5cqcOHD6tXr176/PPPL/h94E318X1e2ZYtW7R06VK98847ioyM1MCBA7Vo0SJJUrt27bR169Y67bdw4cKLjn/77bd12q+yHTt26MMPP1T37t11++23649//KOeeuqpejkn6ciRI4qMjDQfjxo1SsHBwebjtm3b6ptvvvFK77i4OI0bN06ffvppvX30VqEh9y01+WjMx8dH77zzjm677TbFxsbWWW+CUB07c+aMXnrpJc2cOVMtWrTQ8uXLvXJi27lOnz6tP/3pT1q2bJm2bNmiqKgoJScna8SIEYqKivJ6/wvp1KmT13YYd9xxh1566SXdddddVY4vXLhQt99+u1d6//znP9eWLVv0s5/9TPn5+fr73/+uO+64wxz/5ptv1KJFizrvu3v3bg0fPlwnT540w/W3336r0aNHq0uXLnrxxReVnJxc530r++abb8yg/dVXX+n222/XkiVL9OCDDyooKMirvSWpd+/e6tq1q3JyctSkSRNNmzatQUJQZd78Pne5XMrIyNDSpUvldrv1q1/9SiUlJVqzZo2io6PNOl9fX3Xr1q1Oe8+bN6/amtatW9dpzwrff/+9GUaCg4PVpEkT9e7d2yu9zmWz2VRQUGD+8D/3dSgsLFSjRo280vvuu+/W0qVLdezYMQ0fPlxOp1M+Pj5e6VVZQ+9bKgfN6oSEhOjdd9+tu+Z1dmzJ4srLy42MjAyjdevWRmRkpPHKK68YZ8+e9Xrfbdu2GY899pgREhJiBAYGGkOHDjWys7ON8vJyr/euiZycHKNDhw5e2fauXbuMgIAAIzEx0di2bZtRVFRkFBUVGZ9++qkxcOBAIyAgwMjLy/NK71dffdVo0qSJMXLkSKNz586Gw+HwGH/uueeM/v3713lff39/Y/To0cZPP/103tjvf/97w263G/369avzvoZhGCtXrjScTqfh5+dnREZGGlOnTjUOHjzoUVPV+Tt1acWKFUZoaKhx1113Gfv37zcmTpxo+Pv7G+PHjzdOnTrl1d4X463v8/79+xt2u90YOnSosW7dOnOfUl8fkzQkm81mHDp0yCguLjaKioqMZs2aGX/961+N4uJij8UbHA6H8dvf/vaC4zNmzDjvPV+X8vPzjWeeecZo27atER4ebp6rU/nj+LrWkPuWhkYQqiNdunQxGjdubEyePNn47rvvznuzeutNa7PZjJtvvtl46aWXjB9++KHOt385PvvsM+Pmm282xo8f77Uea9euNVq2bGmeUFmxtGzZ0vjzn//stb6GYRjLli0zBgwYYIwZM8ZwuVweY2PGjDHefffdOu/5wQcfXHS84pwZb2jUqJExYMAAY+3atUZZWZm5vuJ8lZ49e3r1HKGBAwcaTZo0MRYuXOix/i9/+YvRsWNHo2PHjsbWrVu91v9CvPl97uvra0yYMMH4+9//7rG+voJQTk6O0blz5yr3XUVFRUZ0dLSxZcsWr/SuOBG9YrnQY2949dVXjcaNGxvr1q07b+z99983GjdubLz66qte6X2u7OxsY+jQoUZgYKDRoUMHY+rUqV75Ba+6fcuRI0e8tm9paHw0Vkf27dsnSZozZ45eeOGF88YNw5CPj0+dX2ZqGIYaNWokf3//C15W7E3Nmzev8rDtyZMndfbsWd1zzz169tlnvda/f//++vrrr5WVlaVDhw7JMAx17NhRffv2VePGjb3Wt7y8XMeOHVNhYaG++eYbzZs3T9OnTzc/GlqyZIlX+vbr16/K9T/99JP++Mc/aunSpdq5c6dXen/zzTcKCwszH1d1vsrixYu90lv610dEn332mTp06OCx/pZbbtHu3bs1ZcoU3XnnnSotLa3z3g31ff7JJ59o6dKliomJUefOnTV8+HANGTKkzvtcyPz58zV69Ogqz7ULDg7WY489prlz53rlI+j/+7//q/Nt1tTo0aO1ceNG3X///erUqZN5rs6BAwd04MABJSYmeu18uLKyMv3ud7/T+++/r9LSUvOjssWLF+vNN9/UsmXL9Pzzz9f5z5KtW7eqb9++8vX1rXLcx8enXj6iawjcR6iObN68uUZ1d955Z532/eSTT7R8+XKtXr1a5eXlSkxM1KhRo7x2bsy5MjIyqnxz2O123XDDDR7nMNS1jRs3auzYsfr000/P21EXFxfrlltuUXp6uldei+eee07PPPOM4uPjFRQUpA0bNmjo0KHmfUfqS1VhJDExUT179vRKv6rOV0lPT9df//pXr36tpX/dQLN9+/YX3Rlv2bLF41ytutKQ3+fSvwLXypUrtWzZMm3fvl1lZWWaO3euRo4cqWbNmnmtb5s2bZSVlaXOnTtXOb5//3717dtX+fn5dd67qkBQ+ZeN+vD222/rj3/8ow4ePChJ6tChg4YOHerVMFqTfcuuXbvUo0ePOu3bunVrtWjRQn/4wx/OuxfVK6+8ookTJ+rWW2/V+vXr67TvFaFBj0ehzpw4ccJYtmyZcccddxg+Pj5Ghw4djNmzZxvfffedV/uePXvWmD17tnHLLbcYsbGxxuTJk+vtXjb333+/MXfu3AuOL1iwwBgwYIBXerdv395IT083H2dnZxv+/v4eHxl5y3fffWfMmjXLaN++vREWFmaMHTu2Xj4qaejzVWw2m1FYWGg+/tWvfnXeR5JWUHFuVEREhBEYGGjcf//9XusVEBBw3nlglR08eNAIDAz0Su8ZM2YYNpvN6Nu3r/HAAw8YgYGBxogRI7zS60rSUPuWK/H2FPWFIFRHzv38uqqlvu7nc/DgQeOpp54yoqKijEaNGnl1R9mQO6vWrVtf9OTBv/3tb3V6r4nK/P39jfz8fI91AQEBXrvBW4WGDCMNfb6Kj4+PRxA6994u3u59pby/K5w9e9Z47733vPr+/tnPfma89957Fxx/5513jHbt2nmld0P+stGQX++G2rdUWLNmjREeHm5069bNsNvtRnx8/EXv13Yt4ByhOvLee+9dcCw3N1cLFy5UeXl5vcylffv2euqpp9SmTRtNnTpVmZmZXuv1xhtvaMmSJXrsscckSR999JESEhL02muvyWbz7p+yq+4SVj8/Px0/ftwrvc+ePavAwECPdY0aNdKZM2e80q/C+vXr9cQTT2jMmDHnnSvjbQ19vkpDevfddy/4kVx9v78r+Pr6asCAARowYIDXetx33316+umnde+99573/X7q1ClNnz5d/fv390rv/Px83Xfffebj+Ph4+fj4eFzW7i0NuT9vqH1LhSvx9hRe19BJ7Fq2f/9+Y8CAAYavr6/x8MMP10uq3rx5s5GUlGQ0bdrUsNvtxqhRo4zc3Fyv9WvI314a8rdVHx8f47777jN++ctfmoufn5/Rt29fj3V1LTc31xg1apTRrFkzo1evXsZLL71kHD9+vF4vpz5x4oSxdOlS49ZbbzVv/T9//nzD7XZ7ta/NZjOOHTtmPm7atKnx1VdfebXnxTTE+7u+uVwuIzIy0oiKijKef/55Y82aNcaaNWuM2bNnG1FRUUZkZKTXPp489+ttGA37Na+vr3dD7VsM48q9PYW3cbK0FxQUFGj69Ol6/fXX5XQ6NWvWLK/+IcSCggLzBneHDh3SLbfcouTkZP3qV79SkyZNvNZX+tdvpS6XSy1btjTXNWvWTJ9//rnatWvn1d7jxo3Tpk2btGPHjip/W+3Vq5f69OlT7d1xL8WIESNqVLd8+fI67y013Mmz5zpw4ICWLl2qP/zhDyoqKtI999xzwT9QerlsNpv69etn3tF77dq1uuuuu877Hq/TG61Vob7f3w3t66+/1pgxY7Rhwwbzj7/6+PjI6XRq8eLFXnufn/v1lqr+ml9rX++G2rckJiZqw4YNmjVrlsaNG2eu37p1qzmnjIwMORyOOu17JSAI1aHi4mLNnDlTL730krp3767nn3/e61dv9evXTx999JGuu+46Pfzwwxo5cmS93pa9IXdWhYWF6tGjh3x9fTV27Fjz/71//34tXrxYZWVl2rVrl8LDw+u895WkPsPIhZSVlWnt2rVatmyZ13o3dPhsiPf3leTHH380b1HRoUMH86+Tewtf7/p16623KiMjo8qP3E+dOqUpU6bo5Zdf9srtKRoaQaiOzJkzR88//7wiIiI0c+ZMPfDAA/XS9z//8z+VnJys/v37X/D+D97U0Durhvpt9UpUH2HEqhrq/Y2GYcWvd3l5ebXndXrr9hQNjSBUR2w2m4KCghQfH3/RQOLtw7hWVd+/rcJaeH9bC19va+GqsTry8MMPX7N33bwaNG/e3Gs3EQR4f1sLX29r4YgQAACwLO/e6AUAAOAKRhACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACWRRACAACW9f8A/ft9KNtPDs8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_train_df['label'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "98114dad-ef24-4a97-8595-19929a327b3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: >"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGkCAYAAADT+zRtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7XElEQVR4nO3de3gU9d3+8TubkAOHTQAhMT/CoQWECIKcwnpq0ciKwUdKsIA8ECGg0oBCWk6WBootKFZOAsYDEKxSgapUiARjKKASOQRRRKGoQFDcgNVkhUICyff3R6/Mw0o4BLKJDO/Xdc0FO9/PzGd2s7u5MzszG2CMMQIAALAZR01vAAAAgD8QcgAAgC0RcgAAgC0RcgAAgC0RcgAAgC0RcgAAgC0RcgAAgC0RcgAAgC0F1fQG1KSysjIdPnxY9erVU0BAQE1vDgAAuAjGGP3www+Kjo6Ww3Hu/TVXdcg5fPiwYmJianozAADAJTh06JCaNGlyzvGrOuTUq1dP0n8fJKfTWcNbAwAALobX61VMTIz1e/xcruqQU/4RldPpJOQAAHCFudChJhx4DAAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbCmoMsXNmzfXwYMHz5r/m9/8RgsWLNDJkyf129/+Vq+++qqKi4vldru1cOFCRUZGWrX5+fkaOXKk/vnPf6pu3bpKSkrSjBkzFBT0f5uyYcMGpaamavfu3YqJidHkyZP1wAMP+PRcsGCBnnrqKXk8HnXo0EHPPPOMunXrVsm7f4H7OzHzspY/8ERCFW0JAACorErtydm2bZu++eYba8rOzpYk3XfffZKksWPHavXq1Vq5cqU2btyow4cPq2/fvtbypaWlSkhIUElJiTZv3qylS5cqIyNDaWlpVs3+/fuVkJCgHj16aOfOnRozZoyGDx+udevWWTXLly9XamqqpkyZoh07dqhDhw5yu906cuTIZT0YAADAPgKMMeZSFx4zZozWrFmjffv2yev1qlGjRlq2bJn69esnSdqzZ4/atm2r3Nxcde/eXWvXrlXv3r11+PBha+9Oenq6JkyYoKNHjyo4OFgTJkxQZmamPvnkE6vPgAEDVFhYqKysLElSXFycunbtqvnz50uSysrKFBMTo9GjR2vixIkXvf1er1fh4eEqKiqS0+k8a5w9OQAA/PRc6Pd3uUs+JqekpEQvv/yyhg0bpoCAAOXl5enUqVOKj4+3atq0aaOmTZsqNzdXkpSbm6v27dv7fHzldrvl9Xq1e/duq+bMdZTXlK+jpKREeXl5PjUOh0Px8fFWzbkUFxfL6/X6TAAAwJ4uOeSsWrVKhYWF1rEyHo9HwcHBioiI8KmLjIyUx+Oxas4MOOXj5WPnq/F6vTpx4oS+/fZblZaWVlhTvo5zmTFjhsLDw60pJiamUvcZAABcOS455CxatEi9evVSdHR0VW6PX02aNElFRUXWdOjQoZreJAAA4CeVOruq3MGDB/XOO+/o9ddft+ZFRUWppKREhYWFPntzCgoKFBUVZdVs3brVZ10FBQXWWPm/5fPOrHE6nQoLC1NgYKACAwMrrClfx7mEhIQoJCSkcncWAABckS5pT86SJUvUuHFjJST834G1nTt3Vq1atZSTk2PN27t3r/Lz8+VyuSRJLpdLu3bt8jkLKjs7W06nU7GxsVbNmesorylfR3BwsDp37uxTU1ZWppycHKsGAACg0ntyysrKtGTJEiUlJflc2yY8PFzJyclKTU1VgwYN5HQ6NXr0aLlcLnXv3l2S1LNnT8XGxmrw4MGaOXOmPB6PJk+erJSUFGsPy8MPP6z58+dr/PjxGjZsmNavX68VK1YoM/P/znRKTU1VUlKSunTpom7dumnOnDk6fvy4hg4dermPBwAAsIlKh5x33nlH+fn5GjZs2Fljs2fPlsPhUGJios/FAMsFBgZqzZo1GjlypFwul+rUqaOkpCRNmzbNqmnRooUyMzM1duxYzZ07V02aNNGLL74ot9tt1fTv319Hjx5VWlqaPB6POnbsqKysrLMORgYAAFevy7pOzpWO6+QAAHDl8ft1cgAAAH7KCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWCDkAAMCWKh1yvv76a/3v//6vGjZsqLCwMLVv317bt2+3xo0xSktL07XXXquwsDDFx8dr3759Puv47rvvNGjQIDmdTkVERCg5OVnHjh3zqfn444916623KjQ0VDExMZo5c+ZZ27Jy5Uq1adNGoaGhat++vd56663K3h0AAGBTlQo533//vW6++WbVqlVLa9eu1aeffqqnn35a9evXt2pmzpypefPmKT09XVu2bFGdOnXkdrt18uRJq2bQoEHavXu3srOztWbNGm3atEkPPvigNe71etWzZ081a9ZMeXl5euqppzR16lQ9//zzVs3mzZs1cOBAJScn68MPP1SfPn3Up08fffLJJ5fzeAAAAJsIMMaYiy2eOHGi3n//fb377rsVjhtjFB0drd/+9rf63e9+J0kqKipSZGSkMjIyNGDAAH322WeKjY3Vtm3b1KVLF0lSVlaW7r77bn311VeKjo7Ws88+q9///vfyeDwKDg62eq9atUp79uyRJPXv31/Hjx/XmjVrrP7du3dXx44dlZ6eflH3x+v1Kjw8XEVFRXI6nWeNN5+YebEPTYUOPJFwWcsDAICzXej3d7lK7cl588031aVLF913331q3LixbrzxRr3wwgvW+P79++XxeBQfH2/NCw8PV1xcnHJzcyVJubm5ioiIsAKOJMXHx8vhcGjLli1WzW233WYFHElyu93au3evvv/+e6vmzD7lNeV9KlJcXCyv1+szAQAAe6pUyPnyyy/17LPPqlWrVlq3bp1GjhypRx55REuXLpUkeTweSVJkZKTPcpGRkdaYx+NR48aNfcaDgoLUoEEDn5qK1nFmj3PVlI9XZMaMGQoPD7emmJiYytx9AABwBalUyCkrK1OnTp00ffp03XjjjXrwwQc1YsSIi/54qKZNmjRJRUVF1nTo0KGa3iQAAOAnlQo51157rWJjY33mtW3bVvn5+ZKkqKgoSVJBQYFPTUFBgTUWFRWlI0eO+IyfPn1a3333nU9NRes4s8e5asrHKxISEiKn0+kzAQAAe6pUyLn55pu1d+9en3n/+te/1KxZM0lSixYtFBUVpZycHGvc6/Vqy5YtcrlckiSXy6XCwkLl5eVZNevXr1dZWZni4uKsmk2bNunUqVNWTXZ2tq677jrrTC6Xy+XTp7ymvA8AALi6VSrkjB07Vh988IGmT5+uzz//XMuWLdPzzz+vlJQUSVJAQIDGjBmjP/3pT3rzzTe1a9cuDRkyRNHR0erTp4+k/+75ueuuuzRixAht3bpV77//vkaNGqUBAwYoOjpaknT//fcrODhYycnJ2r17t5YvX665c+cqNTXV2pZHH31UWVlZevrpp7Vnzx5NnTpV27dv16hRo6rooQEAAFeyoMoUd+3aVW+88YYmTZqkadOmqUWLFpozZ44GDRpk1YwfP17Hjx/Xgw8+qMLCQt1yyy3KyspSaGioVfPKK69o1KhRuuOOO+RwOJSYmKh58+ZZ4+Hh4Xr77beVkpKizp0765prrlFaWprPtXRuuukmLVu2TJMnT9Zjjz2mVq1aadWqVWrXrt3lPB4AAMAmKnWdHLvhOjkAAFx5/HKdHAAAgCsFIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANhSpULO1KlTFRAQ4DO1adPGGj958qRSUlLUsGFD1a1bV4mJiSooKPBZR35+vhISElS7dm01btxY48aN0+nTp31qNmzYoE6dOikkJEQtW7ZURkbGWduyYMECNW/eXKGhoYqLi9PWrVsrc1cAAIDNVXpPzvXXX69vvvnGmt577z1rbOzYsVq9erVWrlypjRs36vDhw+rbt681XlpaqoSEBJWUlGjz5s1aunSpMjIylJaWZtXs379fCQkJ6tGjh3bu3KkxY8Zo+PDhWrdunVWzfPlypaamasqUKdqxY4c6dOggt9utI0eOXOrjAAAAbCbAGGMutnjq1KlatWqVdu7cedZYUVGRGjVqpGXLlqlfv36SpD179qht27bKzc1V9+7dtXbtWvXu3VuHDx9WZGSkJCk9PV0TJkzQ0aNHFRwcrAkTJigzM1OffPKJte4BAwaosLBQWVlZkqS4uDh17dpV8+fPlySVlZUpJiZGo0eP1sSJEy/6znu9XoWHh6uoqEhOp/Os8eYTMy96XRU58ETCZS0PAADOdqHf3+UqvSdn3759io6O1s9+9jMNGjRI+fn5kqS8vDydOnVK8fHxVm2bNm3UtGlT5ebmSpJyc3PVvn17K+BIktvtltfr1e7du62aM9dRXlO+jpKSEuXl5fnUOBwOxcfHWzXnUlxcLK/X6zMBAAB7qlTIiYuLU0ZGhrKysvTss89q//79uvXWW/XDDz/I4/EoODhYERERPstERkbK4/FIkjwej0/AKR8vHztfjdfr1YkTJ/Ttt9+qtLS0wprydZzLjBkzFB4ebk0xMTGVufsAAOAKElSZ4l69eln/v+GGGxQXF6dmzZppxYoVCgsLq/KNq2qTJk1Samqqddvr9RJ0AACwqcs6hTwiIkKtW7fW559/rqioKJWUlKiwsNCnpqCgQFFRUZKkqKios862Kr99oRqn06mwsDBdc801CgwMrLCmfB3nEhISIqfT6TMBAAB7uqyQc+zYMX3xxRe69tpr1blzZ9WqVUs5OTnW+N69e5Wfny+XyyVJcrlc2rVrl89ZUNnZ2XI6nYqNjbVqzlxHeU35OoKDg9W5c2efmrKyMuXk5Fg1AAAAlQo5v/vd77Rx40YdOHBAmzdv1q9+9SsFBgZq4MCBCg8PV3JyslJTU/XPf/5TeXl5Gjp0qFwul7p37y5J6tmzp2JjYzV48GB99NFHWrdunSZPnqyUlBSFhIRIkh5++GF9+eWXGj9+vPbs2aOFCxdqxYoVGjt2rLUdqampeuGFF7R06VJ99tlnGjlypI4fP66hQ4dW4UMDAACuZJU6Juerr77SwIED9e9//1uNGjXSLbfcog8++ECNGjWSJM2ePVsOh0OJiYkqLi6W2+3WwoULreUDAwO1Zs0ajRw5Ui6XS3Xq1FFSUpKmTZtm1bRo0UKZmZkaO3as5s6dqyZNmujFF1+U2+22avr376+jR48qLS1NHo9HHTt2VFZW1lkHIwMAgKtXpa6TYzdcJwcAgCuP366TAwAAcCUg5AAAAFsi5AAAAFsi5AAAAFsi5AAAAFuq1CnkqD6c2QUAwOVhTw4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlvtYBZ+ErJQAAdsCeHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuXFXKeeOIJBQQEaMyYMda8kydPKiUlRQ0bNlTdunWVmJiogoICn+Xy8/OVkJCg2rVrq3Hjxho3bpxOnz7tU7NhwwZ16tRJISEhatmypTIyMs7qv2DBAjVv3lyhoaGKi4vT1q1bL+fuAAAAG7nkkLNt2zY999xzuuGGG3zmjx07VqtXr9bKlSu1ceNGHT58WH379rXGS0tLlZCQoJKSEm3evFlLly5VRkaG0tLSrJr9+/crISFBPXr00M6dOzVmzBgNHz5c69ats2qWL1+u1NRUTZkyRTt27FCHDh3kdrt15MiRS71LAADARi4p5Bw7dkyDBg3SCy+8oPr161vzi4qKtGjRIs2aNUu33367OnfurCVLlmjz5s364IMPJElvv/22Pv30U7388svq2LGjevXqpccff1wLFixQSUmJJCk9PV0tWrTQ008/rbZt22rUqFHq16+fZs+ebfWaNWuWRowYoaFDhyo2Nlbp6emqXbu2Fi9efDmPBwAAsIlLCjkpKSlKSEhQfHy8z/y8vDydOnXKZ36bNm3UtGlT5ebmSpJyc3PVvn17RUZGWjVut1ter1e7d++2an68brfbba2jpKREeXl5PjUOh0Px8fFWTUWKi4vl9Xp9JgAAYE9BlV3g1Vdf1Y4dO7Rt27azxjwej4KDgxUREeEzPzIyUh6Px6o5M+CUj5ePna/G6/XqxIkT+v7771VaWlphzZ49e8657TNmzNAf//jHi7ujAADgilapPTmHDh3So48+qldeeUWhoaH+2ia/mTRpkoqKiqzp0KFDNb1JAADATyoVcvLy8nTkyBF16tRJQUFBCgoK0saNGzVv3jwFBQUpMjJSJSUlKiws9FmuoKBAUVFRkqSoqKizzrYqv32hGqfTqbCwMF1zzTUKDAyssKZ8HRUJCQmR0+n0mQAAgD1VKuTccccd2rVrl3bu3GlNXbp00aBBg6z/16pVSzk5OdYye/fuVX5+vlwulyTJ5XJp165dPmdBZWdny+l0KjY21qo5cx3lNeXrCA4OVufOnX1qysrKlJOTY9UAAICrW6WOyalXr57atWvnM69OnTpq2LChNT85OVmpqalq0KCBnE6nRo8eLZfLpe7du0uSevbsqdjYWA0ePFgzZ86Ux+PR5MmTlZKSopCQEEnSww8/rPnz52v8+PEaNmyY1q9frxUrVigzM9Pqm5qaqqSkJHXp0kXdunXTnDlzdPz4cQ0dOvSyHhAAAGAPlT7w+EJmz54th8OhxMREFRcXy+12a+HChdZ4YGCg1qxZo5EjR8rlcqlOnTpKSkrStGnTrJoWLVooMzNTY8eO1dy5c9WkSRO9+OKLcrvdVk3//v119OhRpaWlyePxqGPHjsrKyjrrYGQAAHB1CjDGmJreiJri9XoVHh6uoqKiCo/PaT4xs4KlLt6BJxIuedmrtTcAABdyod/f5fjuKgAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuEHAAAYEuVCjnPPvusbrjhBjmdTjmdTrlcLq1du9YaP3nypFJSUtSwYUPVrVtXiYmJKigo8FlHfn6+EhISVLt2bTVu3Fjjxo3T6dOnfWo2bNigTp06KSQkRC1btlRGRsZZ27JgwQI1b95coaGhiouL09atWytzVwAAgM1VKuQ0adJETzzxhPLy8rR9+3bdfvvtuvfee7V7925J0tixY7V69WqtXLlSGzdu1OHDh9W3b19r+dLSUiUkJKikpESbN2/W0qVLlZGRobS0NKtm//79SkhIUI8ePbRz506NGTNGw4cP17p166ya5cuXKzU1VVOmTNGOHTvUoUMHud1uHTly5HIfDwAAYBMBxhhzOSto0KCBnnrqKfXr10+NGjXSsmXL1K9fP0nSnj171LZtW+Xm5qp79+5au3atevfurcOHDysyMlKSlJ6ergkTJujo0aMKDg7WhAkTlJmZqU8++cTqMWDAABUWFiorK0uSFBcXp65du2r+/PmSpLKyMsXExGj06NGaOHHiRW+71+tVeHi4ioqK5HQ6zxpvPjHzkh8XSTrwRMIlL3u19gYA4EIu9Pu73CUfk1NaWqpXX31Vx48fl8vlUl5enk6dOqX4+Hirpk2bNmratKlyc3MlSbm5uWrfvr0VcCTJ7XbL6/Vae4Nyc3N91lFeU76OkpIS5eXl+dQ4HA7Fx8dbNedSXFwsr9frMwEAAHuqdMjZtWuX6tatq5CQED388MN64403FBsbK4/Ho+DgYEVERPjUR0ZGyuPxSJI8Ho9PwCkfLx87X43X69WJEyf07bffqrS0tMKa8nWcy4wZMxQeHm5NMTExlb37AADgClHpkHPddddp586d2rJli0aOHKmkpCR9+umn/ti2Kjdp0iQVFRVZ06FDh2p6kwAAgJ8EVXaB4OBgtWzZUpLUuXNnbdu2TXPnzlX//v1VUlKiwsJCn705BQUFioqKkiRFRUWddRZU+dlXZ9b8+IysgoICOZ1OhYWFKTAwUIGBgRXWlK/jXEJCQhQSElLZuwwAAK5Al32dnLKyMhUXF6tz586qVauWcnJyrLG9e/cqPz9fLpdLkuRyubRr1y6fs6Cys7PldDoVGxtr1Zy5jvKa8nUEBwerc+fOPjVlZWXKycmxagAAACq1J2fSpEnq1auXmjZtqh9++EHLli3Thg0btG7dOoWHhys5OVmpqalq0KCBnE6nRo8eLZfLpe7du0uSevbsqdjYWA0ePFgzZ86Ux+PR5MmTlZKSYu1hefjhhzV//nyNHz9ew4YN0/r167VixQplZv7fGT+pqalKSkpSly5d1K1bN82ZM0fHjx/X0KFDq/ChAQAAV7JKhZwjR45oyJAh+uabbxQeHq4bbrhB69at05133ilJmj17thwOhxITE1VcXCy3262FCxdaywcGBmrNmjUaOXKkXC6X6tSpo6SkJE2bNs2qadGihTIzMzV27FjNnTtXTZo00Ysvvii3223V9O/fX0ePHlVaWpo8Ho86duyorKyssw5GBgAAV6/Lvk7OlYzr5Pz0egMAcCF+v04OAADATxkhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2FKlQs6MGTPUtWtX1atXT40bN1afPn20d+9en5qTJ08qJSVFDRs2VN26dZWYmKiCggKfmvz8fCUkJKh27dpq3Lixxo0bp9OnT/vUbNiwQZ06dVJISIhatmypjIyMs7ZnwYIFat68uUJDQxUXF6etW7dW5u4AAAAbq1TI2bhxo1JSUvTBBx8oOztbp06dUs+ePXX8+HGrZuzYsVq9erVWrlypjRs36vDhw+rbt681XlpaqoSEBJWUlGjz5s1aunSpMjIylJaWZtXs379fCQkJ6tGjh3bu3KkxY8Zo+PDhWrdunVWzfPlypaamasqUKdqxY4c6dOggt9utI0eOXM7jAQAAbCLAGGMudeGjR4+qcePG2rhxo2677TYVFRWpUaNGWrZsmfr16ydJ2rNnj9q2bavc3Fx1795da9euVe/evXX48GFFRkZKktLT0zVhwgQdPXpUwcHBmjBhgjIzM/XJJ59YvQYMGKDCwkJlZWVJkuLi4tS1a1fNnz9fklRWVqaYmBiNHj1aEydOvKjt93q9Cg8PV1FRkZxO51njzSdmXupDI0k68ETCJS97tfYGAOBCLvT7u9xlHZNTVFQkSWrQoIEkKS8vT6dOnVJ8fLxV06ZNGzVt2lS5ubmSpNzcXLVv394KOJLkdrvl9Xq1e/duq+bMdZTXlK+jpKREeXl5PjUOh0Px8fFWTUWKi4vl9Xp9JgAAYE+XHHLKyso0ZswY3XzzzWrXrp0kyePxKDg4WBERET61kZGR8ng8Vs2ZAad8vHzsfDVer1cnTpzQt99+q9LS0gprytdRkRkzZig8PNyaYmJiKn/HAQDAFeGSQ05KSoo++eQTvfrqq1W5PX41adIkFRUVWdOhQ4dqepMAAICfBF3KQqNGjdKaNWu0adMmNWnSxJofFRWlkpISFRYW+uzNKSgoUFRUlFXz47Ogys++OrPmx2dkFRQUyOl0KiwsTIGBgQoMDKywpnwdFQkJCVFISEjl7zAAALjiVGpPjjFGo0aN0htvvKH169erRYsWPuOdO3dWrVq1lJOTY83bu3ev8vPz5XK5JEkul0u7du3yOQsqOztbTqdTsbGxVs2Z6yivKV9HcHCwOnfu7FNTVlamnJwcqwYAAFzdKrUnJyUlRcuWLdM//vEP1atXzzr+JTw8XGFhYQoPD1dycrJSU1PVoEEDOZ1OjR49Wi6XS927d5ck9ezZU7GxsRo8eLBmzpwpj8ejyZMnKyUlxdrL8vDDD2v+/PkaP368hg0bpvXr12vFihXKzPy/s35SU1OVlJSkLl26qFu3bpozZ46OHz+uoUOHVtVjAwAArmCVCjnPPvusJOmXv/ylz/wlS5bogQcekCTNnj1bDodDiYmJKi4ultvt1sKFC63awMBArVmzRiNHjpTL5VKdOnWUlJSkadOmWTUtWrRQZmamxo4dq7lz56pJkyZ68cUX5Xa7rZr+/fvr6NGjSktLk8fjUceOHZWVlXXWwcgAAODqdFnXybnScZ0cegMArjzVcp0cAACAnypCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsCVCDgAAsKWgmt4A4Kei+cTMy1r+wBMJVbQlAICqwJ4cAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS4QcAABgS5UOOZs2bdI999yj6OhoBQQEaNWqVT7jxhilpaXp2muvVVhYmOLj47Vv3z6fmu+++06DBg2S0+lURESEkpOTdezYMZ+ajz/+WLfeeqtCQ0MVExOjmTNnnrUtK1euVJs2bRQaGqr27dvrrbfequzdAQAANlXpkHP8+HF16NBBCxYsqHB85syZmjdvntLT07VlyxbVqVNHbrdbJ0+etGoGDRqk3bt3Kzs7W2vWrNGmTZv04IMPWuNer1c9e/ZUs2bNlJeXp6eeekpTp07V888/b9Vs3rxZAwcOVHJysj788EP16dNHffr00SeffFLZuwQAAGwoqLIL9OrVS7169apwzBijOXPmaPLkybr33nslSS+99JIiIyO1atUqDRgwQJ999pmysrK0bds2denSRZL0zDPP6O6779Zf/vIXRUdH65VXXlFJSYkWL16s4OBgXX/99dq5c6dmzZplhaG5c+fqrrvu0rhx4yRJjz/+uLKzszV//nylp6df0oMBAADso0qPydm/f788Ho/i4+OteeHh4YqLi1Nubq4kKTc3VxEREVbAkaT4+Hg5HA5t2bLFqrntttsUHBxs1bjdbu3du1fff/+9VXNmn/Ka8j4VKS4ultfr9ZkAAIA9VWnI8Xg8kqTIyEif+ZGRkdaYx+NR48aNfcaDgoLUoEEDn5qK1nFmj3PVlI9XZMaMGQoPD7emmJiYyt5FAABwhbiqzq6aNGmSioqKrOnQoUM1vUkAAMBPqjTkREVFSZIKCgp85hcUFFhjUVFROnLkiM/46dOn9d133/nUVLSOM3ucq6Z8vCIhISFyOp0+EwAAsKcqDTktWrRQVFSUcnJyrHler1dbtmyRy+WSJLlcLhUWFiovL8+qWb9+vcrKyhQXF2fVbNq0SadOnbJqsrOzdd1116l+/fpWzZl9ymvK+wAAgKtbpUPOsWPHtHPnTu3cuVPSfw823rlzp/Lz8xUQEKAxY8boT3/6k958803t2rVLQ4YMUXR0tPr06SNJatu2re666y6NGDFCW7du1fvvv69Ro0ZpwIABio6OliTdf//9Cg4OVnJysnbv3q3ly5dr7ty5Sk1Ntbbj0UcfVVZWlp5++mnt2bNHU6dO1fbt2zVq1KjLf1QAAMAVr9KnkG/fvl09evSwbpcHj6SkJGVkZGj8+PE6fvy4HnzwQRUWFuqWW25RVlaWQkNDrWVeeeUVjRo1SnfccYccDocSExM1b948azw8PFxvv/22UlJS1LlzZ11zzTVKS0vzuZbOTTfdpGXLlmny5Ml67LHH1KpVK61atUrt2rW7pAcCAADYS6VDzi9/+UsZY845HhAQoGnTpmnatGnnrGnQoIGWLVt23j433HCD3n333fPW3HfffbrvvvvOv8EAAOCqdFWdXQUAAK4eld6TA6DqNZ+YeVnLH3gioYq2BADsgz05AADAlgg5AADAlgg5AADAljgmB7jK1eTxQByLBMCf2JMDAABsiT05AK5K7EUC7I89OQAAwJYIOQAAwJYIOQAAwJYIOQAAwJYIOQAAwJYIOQAAwJYIOQAAwJYIOQAAwJYIOQAAwJYIOQAAwJb4WgcAqGZ8pQRQPdiTAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImzqwDgKsKZXbiasCcHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEtfJAQBUC67Rg+rGnhwAAGBL7MkBANgee5GuTuzJAQAAtkTIAQAAtkTIAQAAtkTIAQAAtnTFh5wFCxaoefPmCg0NVVxcnLZu3VrTmwQAAH4CruiQs3z5cqWmpmrKlCnasWOHOnToILfbrSNHjtT0pgEAgBp2RYecWbNmacSIERo6dKhiY2OVnp6u2rVra/HixTW9aQAAoIZdsdfJKSkpUV5eniZNmmTNczgcio+PV25uboXLFBcXq7i42LpdVFQkSfJ6vRXWlxX/57K28VzrvRj0pje96U1ve/RuN2XdZfX+5I/uy1rejsp/HsaY8xeaK9TXX39tJJnNmzf7zB83bpzp1q1bhctMmTLFSGJiYmJiYmKywXTo0KHzZoUrdk/OpZg0aZJSU1Ot22VlZfruu+/UsGFDBQQEVGpdXq9XMTExOnTokJxOZ1VvKr3pTW9605ve9D4HY4x++OEHRUdHn7fuig0511xzjQIDA1VQUOAzv6CgQFFRURUuExISopCQEJ95ERERl7UdTqez2p8c9KY3velNb3pf7b3Dw8MvWHPFHngcHByszp07Kycnx5pXVlamnJwcuVyuGtwyAADwU3DF7smRpNTUVCUlJalLly7q1q2b5syZo+PHj2vo0KE1vWkAAKCGXdEhp3///jp69KjS0tLk8XjUsWNHZWVlKTIy0u+9Q0JCNGXKlLM+/qoO9KY3velNb3rT+8ICjLnQ+VcAAABXniv2mBwAAIDzIeQAAABbIuQAAABbIuQAAABbIuQAAABbIuT8hPXr109ZWVkX/gIy4DIdOXLkgjXvvvuuX3p36dJF6enpl/UFiKh6hYWFWrZsWU1vBqpITb7GaxIh5yfs+++/V0JCgpo2baq0tDR9+eWXNb1J1Wbfvn0aOHBghb/4ioqKdP/991f747Fx40a99dZb+v7776u1b3Vo166d/v73v1c4duLECT3yyCO64447/NK7Q4cOGj9+vK699loNHjxYGzZs8Eufn5qPP/74gtOnn36q7777rka27+DBgxo8eHCN9JZ0zuejXRljtHbtWvXr188v66/J13hN4jo5F+Gll166qLohQ4ZUee+DBw9qyZIleumll3Tw4EH94he/0PDhw5WYmOj3iyi9+eabFc4PDw9X69atde211/qt94MPPqiIiAjNnDmzwvEJEybI6/Xq2WefrfLeTz75pI4dO6bHH39c0n/ffHr16qW3335bktS4cWPl5OTo+uuvr/Lef/jDHzRlyhQFBVV8nc78/HwlJycrOzu7Svs+/fTT+sMf/qB7771XCxcuVP369SX99y+7oUOHyuFwaMmSJbr55purtG+5//znP1qxYoUyMjL07rvvqkWLFho2bJiSkpL0//7f//NLz3I19Tx3OBwKCAi44J7agIAAdejQQS+99JLatWvnl22pyEcffaROnTqptLTUL+s/ffq09uzZo+DgYLVu3dqa/49//ENpaWnas2ePiouLq7zvF198oT//+c9avHixJKlp06Y6duyYNR4YGKj33ntP1113XZX3rsj+/fu1ePFiZWRk6OjRo4qPj9eaNWuqvE9Nvsb79u17wZqgoCBFRUXpzjvv1D333FN1zc/7HeUwxhgTERFxzql+/fomODjYOBwOv29HTk6OGTRokKldu7apX7+++c1vfmO2b9/ut34BAQHnnBwOh7n//vvN8ePH/dK7devWZuvWrecc3759u2ndurVfet94443m1VdftW6vWLHChIWFmffee8/8+9//NgkJCea+++7zS++YmBjTsWNHs2vXrrPG0tPTTb169cxdd93ll967d+82Xbp0Mddee61ZuXKleeSRR0xQUJAZPXq0+c9//uOXnhX5/PPPze9//3vTtGlTExQUZO6++27z2muv+a1fTT3PDxw4cMHp448/Nrm5uaZv377mlltuqfJtOJ+dO3f67X1t165dplmzZsbhcBiHw2F+9atfGY/HY2677TbToEEDM2HCBHPo0CG/9H700UfNxIkTrdt169Y1M2fONBkZGSYjI8P06tXLPPTQQ37pXe7kyZPm5ZdfNj169DC1atUyDofDzJo1yxQVFfm1b029xpOSkswDDzxw3mnIkCHmrrvuMmFhYeYPf/hDlfUm5FyGw4cPm4ceesjUqlXLuN3uauvr9XpNenq6adCggQkMDKy2vuUKCwtNTk6OadOmjZk0aZJfeoSGhpoDBw6cc/zAgQMmLCzML70jIiLMp59+at1+4IEHzODBg63bubm5pkmTJn7pXVRUZAYPHmxCQkLM9OnTTWlpqTl48KC54447jNPpNM8995xf+pY7ffq06d+/v3E4HKZu3bpmw4YNfu13PmVlZWblypWmQYMG1fJHxI9Vx/O8Il6v1zz33HOmW7du1v3et2+fqV27drVtgzH+DTl33323ueOOO8zq1avN/fffbwICAkybNm3MU0895fdA3a5dO7Nlyxbrdt26dc0XX3xh3d6wYYNp2bKlX3pv377djBw50kRERJguXbqYuXPnGo/HY4KCgszu3bv90vPHfkqv8YqsXr3axMTEVNn6rujvrqopP/zwg5588knNnTtX119/vdatW6cePXpUS+/9+/crIyNDGRkZKioqUnx8fLX0PVN4eLhuv/12zZ49W2PGjNH06dP90uOLL75Qs2bNKhz//PPP5XQ6q7yv9N/d6Gd+FJibm6sxY8ZYt6Ojo/Xtt9/6pbfT6dRLL72kxMREPfTQQ1q+fLn279+vbt266eOPPz7n41EVTp06pSlTpuj1119X//79lZWVpenTp+vnP/+5mjRp4re+FdmwYYOWLFmi1157TUFBQRoxYkS19peq53l+pk2bNmnRokV67bXXFB0drb59+2r+/PmSpBYtWmjz5s1V2m/evHnnHf/666+rtN+Ztm3bprffflsdO3bUrbfeqr/97W967LHHquUYoAMHDig6Otq6PXz4cIWHh1u3mzdvrq+++sovvePi4jR69Gh98MEH1fZx2Jlq6jV+MR9XBQQE6LXXXtMtt9yiLl26VFlvQk4lnDp1Ss8884ymT5+uhg0basmSJX47SOxMJ0+e1N///nctXrxYmzZtUkxMjJKTkzV06FDFxMT4vf+5tGnTxm9vBrfddpueeeYZ3X777RWOz5s3T7feeqtfev/85z/Xpk2b9LOf/Uz5+fn617/+pdtuu80a/+qrr9SwYUO/9C7XvXt3tW/fXjk5OapTp44mT57s14Czc+dODR48WMePH7dC+9dff60RI0aoXbt2evrpp5WcnOy3/tJ/H9fyAP/ll1/q1ltv1cKFC3XfffcpLCzMr73Px5/Pc4/Ho4yMDC1atEher1e//vWvVVxcrFWrVik2NtaqCwwMVIcOHaq09+zZsy9Y07Rp0yrtWe7bb7+1gkZ4eLjq1Kmj7t27+6XXjzkcDh0+fNj6pf7jx6GgoEC1atXyS+877rhDixYt0pEjRzR48GC53W4FBAT4pdeP1eRr/MwQeSERERF6/fXXq655le0TsrGysjKTkZFhmjZtaqKjo81zzz1nTp8+7fe+W7ZsMQ899JCJiIgwoaGhZuDAgSY7O9uUlZX5vffFyMnJMa1atfLLunfs2GFCQkJMYmKi2bJliyksLDSFhYXmgw8+MH379jUhISEmLy/PL72ff/55U6dOHTNs2DDTtm1b43K5fMYff/xx07t3b7/0NsaYZcuWmQYNGpjbb7/d7Nmzx4wbN84EBwebMWPGmBMnTvilZ3BwsBkxYoT54Ycfzhp74YUXjNPpNL169fJL7+XLlxu3222CgoJMdHS0mTRpktm3b59PTUXHKFUXfz3Pe/fubZxOpxk4cKBZs2aN9Z5SnR9d1BSHw2E+//xzU1RUZAoLC029evXMRx99ZIqKinwmf3C5XObPf/7zOcenTZt21mu+KuXn55upU6ea5s2bm8jISOu4mDM/IveHmnyN1yRCzkVo166dqV27tpkwYYL55ptvznoh+usF6XA4zI033mieeeYZ891331X5+i/Hhx9+aG688UYzZswYv/VYvXq1adSokXVwYvnUqFEj849//MNvfY0xZvHixaZPnz5m5MiRxuPx+IyNHDnSvP76637p27dvX1OnTh0zb948n/nvv/++ad26tWndurXZvHlzlfd96623zjteflyQP9SqVcv06dPHrF692pSWllrzy49N6dq1a40ck2OMf5/ngYGBZuzYseZf//qXz/zqCjk5OTmmbdu2Fb53FRYWmtjYWLNp0ya/9C4/qLt8Otdtf3j++edN7dq1zZo1a84ae/PNN03t2rXN888/75feP5adnW0GDhxoQkNDTatWrcykSZP89sfbhV7jBw4c8NtrvCbxcdVF2L17tyRp5syZeuqpp84aN8YoICCgyk+1NMaoVq1aCg4OPucpxf5Uv379CnelHj9+XKdPn9add96pP/7xj37r37t3bx08eFBZWVn6/PPPZYxR69at1bNnT9WuXdtvfcvKynTkyBEVFBToq6++0uzZszVlyhTrI5OFCxf6rbfH49GHH36oVq1a+cy/6aabtHPnTk2cOFG/+MUvVFJSUqV9e/XqVeH8H374QX/729+0aNEibd++vUp7lvvqq6/UuHFj63ZFx6YsWLDAL72lmnuev/fee1q0aJE6d+6stm3bavDgwRowYECV9zmXOXPmaMSIERUe2xYeHq6HHnpIs2bN8svHwv/85z+rfJ0Xa8SIEVq/fr3uuecetWnTxjo2Zu/evdq7d68SExP9dgxYaWmp/vKXv+jNN99USUmJ9fHVggUL9PLLL2vx4sV68skn/XLa/ubNm9WzZ08FBgZWOB4QEFBtH51VJ66TcxE2btx4UXW/+MUvqrTve++9pyVLlmjlypUqKytTYmKihg8f7rdjUX4sIyOjwie90+nUdddd53PMQFVbv369Ro0apQ8++OCsN+GioiLddNNNSk9P98tj8fjjj2vq1KmKj49XWFiY1q1bp4EDB1rX1fCnffv2qWXLlud9s9m0aZPPMUL+UFHQSExMVNeuXf3Sr6JjU9LT0/XRRx/59Xkm1ezzXPpvmFq+fLkWL16srVu3qrS0VLNmzdKwYcNUr149v/Vt1qyZsrKy1LZt2wrH9+zZo549eyo/P7/Ke1f0y/7MPySqw6uvvqq//e1v2rdvnySpVatWGjhwoF+D5sW8t+zYsUOdOnWq8t5NmzZVw4YN9de//vWs6y0999xzGjdunG6++WatXbu2ynvXqBrdj4SLcuzYMbN48WJz2223mYCAANOqVSvzxBNPmG+++cavfU+fPm2eeOIJc9NNN5kuXbqYCRMmVNv1Uu655x4za9asc47PnTvX9OnTxy+9W7ZsadLT063b2dnZJjg42OejFH9xOBymoKDAuv3rX//6rI/L/OWbb74xM2bMMC1btjSNGzc2o0aNqpaPTq7mY1N+rPwYrKioKBMaGmruuecev/UKCQk569inM+3bt8+Ehob6pfe0adOMw+EwPXv2NPfee68JDQ01Q4cO9Uuvn5KafG+p6ctT1BRCzkX48efFFU3Vdb2affv2mccee8zExMSYWrVq+fVNsCbfiJo2bXreA/E+++yzKr2WwpmCg4NNfn6+z7yQkBC/XZzsTAEBAT4h58fX8PCXmgwaNX1syk/p9V3u9OnT5o033vDr6/tnP/uZeeONN845/tprr5kWLVr4pXdN/rKvyZ93Tb63lFu1apWJjIw0HTp0ME6n08THx5/3mmRXOo7JuQhvvPHGOcdyc3M1b948lZWVVcu2tGzZUo899piaNWumSZMmKTMz02+9XnrpJS1cuFAPPfSQJOmdd95RQkKCXnzxRTkc/v3aswudxhkUFKSjR4/6pffp06cVGhrqM69WrVo6deqUX/r9FKxdu1aPPPKIRo4cedbxQP5W08emvP766+f8eLC6X9/lAgMD1adPH/Xp08dvPe6++2794Q9/0F133XXW8/3EiROaMmWKevfu7Zfe+fn5uvvuu63b8fHxCggI8Dm1219q8v38p/DeUt2Xp6hxNZ2yrlR79uwxffr0MYGBgWbIkCHVkoQ3btxokpKSTN26dY3T6TTDhw83ubm5futXk3911ORfmQEBAebuu+82v/rVr6wpKCjI9OzZ02eePzgcDnPkyBHrdt26dc2XX37pl15nys3NNcOHDzf16tUz3bp1M88884w5evRotX5kdOzYMbNo0SJz8803W5e6nzNnjvF6vdXS/0w18fqubh6Px0RHR5uYmBjz5JNPmlWrVplVq1aZJ554wsTExJjo6Gi/fVT64+e5MdX3XK9Idf28a/K9xZiauTxFTePA40o6fPiwpkyZoqVLl8rtdmvGjBl+/dK8w4cPWxdI+/zzz3XTTTcpOTlZv/71r1WnTh2/9ZX++9ekx+NRo0aNrHn16tXTxx9/rBYtWvi19+jRo7VhwwZt27atwr8yu3Xrph49elzwqq2XYujQoRdVt2TJkirv7XA41KtXL+uKy6tXr9btt99+1s+6Si+WdYaaOgj2x/bu3atFixbpr3/9qwoLC3XnnXee84s0q1J1v75r2sGDBzVy5EitW7fO+qLQgIAAud1uLViwwG+v8x8/z6WKn+v+ep6Xq+6fd02+tyQmJmrdunWaMWOGRo8ebc3fvHmztV0ZGRlyuVxV3rsmEXIuUlFRkaZPn65nnnlGHTt21JNPPun3s5x69eqld955R9dcc42GDBmiYcOGVeulwGvyjaigoECdOnVSYGCgRo0aZd3vPXv2aMGCBSotLdWOHTsUGRlZ5b1rUk2+Cf5YTQWNM5WWlmr16tVavHixX3vXxOv7p+T777+3LtPQqlUr6xuq/aWmn+dX48/75ptvVkZGRoUfR584cUITJ07Us88+W+WXp6hphJyLMHPmTD355JOKiorS9OnTde+991ZL3//5n/9RcnKyevfufc5rG/hTTb8R1dRfmfBVXUGjptTU6xs142r9eZeVlV3wWMrquDxFdSPkXASHw6GwsDDFx8efN2z4e9fq1aq6/8rE1YXX99WFn/fVhbOrLsKQIUNseSXIK0X9+vX9dhE6gNf31YWf99WFPTkAAMCW/HuxEwAAgBpCyAEAALZEyAEAALZEyAEAALZEyAEAALZEyAEAALZEyAEAALZEyAEAALb0/wFQQS1kFHKAPAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_eval_df['label'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cb4836eb-a374-4679-9577-d0b0475b96af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['NN', 'VV', 'PU', 'FX', 'AJ', 'NU', 'CL', 'CC', 'NG', 'PS', 'AX',\n",
       "       'AV', 'PR', 'PA', 'IJ', 'XX'], dtype=object)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = new_train_df['label'].unique()\n",
    "id2label = {id:label for id, label in enumerate(labels)}\n",
    "label2id = {label:id for id, label in enumerate(labels)}\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "10c91fd4-f823-48c9-b036-45a2d5e06a74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ธ.กสิกรไทย</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>จับ</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>มือ</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>เวอริไซน์</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>_</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         text  label\n",
       "0  ธ.กสิกรไทย      0\n",
       "1         จับ      1\n",
       "2         มือ      0\n",
       "3   เวอริไซน์      0\n",
       "4           _      2"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_train_df['label'] = new_train_df['label'].apply(lambda label:label2id[label])\n",
    "new_train_df['text'] = new_train_df['text'].apply(lambda text:' '.join(text.split()).lower())\n",
    "new_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4860edee-9509-41c8-9925-eeb89277e973",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>นายก</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ฯ</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>_</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ย้ำ</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>_</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   text  label\n",
       "0  นายก      0\n",
       "1     ฯ      2\n",
       "2     _      2\n",
       "3   ย้ำ      1\n",
       "4     _      2"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_eval_df['label'] = new_eval_df['label'].apply(lambda label:label2id[label])\n",
    "new_eval_df['text'] = new_eval_df['text'].apply(lambda text:' '.join(text.split()).lower())\n",
    "new_eval_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a718f7d9-e85a-4b13-806e-5574790173b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_labels = ['AJ', 'AV', 'AX', 'CC', 'CL', 'FX', 'IJ', 'NG', 'NN', 'NU', 'PA', 'PR', 'PS', 'PU', 'VV', 'XX']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36d3f16d-812d-4d5c-a363-fb6613c400a4",
   "metadata": {},
   "source": [
    "### เริ่มใช้ data จากอันนี้"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a59f74a0-a119-41e3-9267-4e538977c931",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset lst20 (/home/superai023/.cache/huggingface/datasets/lst20/default-data_dir=LST20/0.0.0/e1b2a921fb011578ab43ddbbf789f3c500d62cb2df8ae4ed4b60bae8e4c0d3ad)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8351b1c65fdc492a964132d5aa7e59cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lst20 = load_dataset(\"lst20\", data_dir=\"LST20\")\n",
    "train_df2 = pd.DataFrame(lst20['train'])\n",
    "validation_df2 = pd.DataFrame(lst20['validation'])\n",
    "test_df2 = pd.DataFrame(lst20['test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bee4c917-f76c-4278-956b-ba4bde19fc91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NN',\n",
       " 'VV',\n",
       " 'PU',\n",
       " 'CC',\n",
       " 'PS',\n",
       " 'AX',\n",
       " 'AV',\n",
       " 'FX',\n",
       " 'NU',\n",
       " 'AJ',\n",
       " 'CL',\n",
       " 'PR',\n",
       " 'NG',\n",
       " 'PA',\n",
       " 'XX',\n",
       " 'IJ']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "POS_TAGS = lst20[\"train\"].features[\"pos_tags\"].feature.names\n",
    "lst20[\"train\"].features[\"pos_tags\"].feature.names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5d5408ea-5816-4138-999c-53d55b7d5f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_df(df):\n",
    "    id=[]\n",
    "    tokens=[]\n",
    "    pos=[]\n",
    "    for index in range(len(df)):\n",
    "        for tok in range(len(df['tokens'][index])):\n",
    "            id.append(index)\n",
    "            tokens.append(df['tokens'][index][tok])\n",
    "            pos.append(POS_TAGS[df['pos_tags'][index][tok]])\n",
    "    return pd.DataFrame({\"sentence_id\": id, \"words\": tokens, \"labels\": pos})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "061f26be-e958-4f90-8b1b-d857b815fefc",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = to_df(train_df2)\n",
    "val_data = to_df(validation_df2)\n",
    "test_data = to_df(test_df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2b400abc-3286-409c-bed7-841a3a7b3a7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>words</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>สลด</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>ผัว</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>เมีย</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>เฒ่า</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>ถูก</td>\n",
       "      <td>AX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>ไฟ</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>ช็อต</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>กอด</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>กัน</td>\n",
       "      <td>AV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>ตาย</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>_</td>\n",
       "      <td>PU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>สังเวย</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>ฝน</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>ตก</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "      <td>PU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>น้ำ</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>ท่วม</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>สอง</td>\n",
       "      <td>NU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>สามี</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>ภรรยา</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>ถูก</td>\n",
       "      <td>AX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1</td>\n",
       "      <td>ไฟฟ้า</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1</td>\n",
       "      <td>ช็อต</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>นอน</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "      <td>กอด</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>กัน</td>\n",
       "      <td>AV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1</td>\n",
       "      <td>ตาย</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>ข้าง</td>\n",
       "      <td>PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>เล้า</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>ไก่</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sentence_id   words labels\n",
       "0             0     สลด     VV\n",
       "1             0     ผัว     NN\n",
       "2             0    เมีย     NN\n",
       "3             0    เฒ่า     NN\n",
       "4             0     ถูก     AX\n",
       "5             0      ไฟ     NN\n",
       "6             0    ช็อต     VV\n",
       "7             0     กอด     VV\n",
       "8             0     กัน     AV\n",
       "9             0     ตาย     VV\n",
       "10            0       _     PU\n",
       "11            0  สังเวย     VV\n",
       "12            0      ฝน     NN\n",
       "13            0      ตก     VV\n",
       "14            0       -     PU\n",
       "15            0     น้ำ     NN\n",
       "16            0    ท่วม     VV\n",
       "17            1     สอง     NU\n",
       "18            1    สามี     NN\n",
       "19            1   ภรรยา     NN\n",
       "20            1     ถูก     AX\n",
       "21            1   ไฟฟ้า     NN\n",
       "22            1    ช็อต     VV\n",
       "23            1     นอน     VV\n",
       "24            1     กอด     VV\n",
       "25            1     กัน     AV\n",
       "26            1     ตาย     VV\n",
       "27            1    ข้าง     PS\n",
       "28            1    เล้า     NN\n",
       "29            1     ไก่     NN"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_data.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "092f25e9-c789-43fb-be5b-81587ab51ccc",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"xlm-roberta-base\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "af25e517-2a72-4d56-baa5-04ffb36f541d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.layer_norm.weight', 'lm_head.dense.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# Configure the model\n",
    "ner_args = NERArgs()\n",
    "ner_args.train_batch_size = 64\n",
    "ner_args.eval_batch_size = 64\n",
    "ner_args.use_multiprocessing = True \n",
    "ner_args.evaluate_during_training = True\n",
    "ner_args.overwrite_output_dir = True\n",
    "ner_args.num_train_epochs = 5\n",
    "ner_args.max_seq_length = 128\n",
    "\n",
    "# Stop Early\n",
    "ner_args.use_early_stopping = True\n",
    "ner_args.early_stopping_delta = 0.01\n",
    "# ner_args.early_stopping_metric = \"mcc\"\n",
    "ner_args.early_stopping_metric_minimize = False\n",
    "ner_args.early_stopping_patience = 5\n",
    "ner_args.evaluate_during_training_steps = 1000\n",
    "\n",
    "model = NERModel(\n",
    "     \"auto\", model_name, args=ner_args, labels=POS_TAGS\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "beeb1c02-43c8-4148-9818-95f03497b901",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2efc68d884a0476ab13ebb94d8d5f4e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/125 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "652cac218eec4457938aa31e94e9e9b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f9f5d428408480e9996b12f75c8abb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 0 of 5:   0%|          | 0/990 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "160a4dc350554f64a88a09817db9f039",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b03d065a40e54bfda50f30d65ec731d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99c790b634c14b4eb928434fe59b46f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 1 of 5:   0%|          | 0/990 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f07ece663b7a42ad90422803e5399b3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "018555d81c7d4a2cadef784c93589a24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b23bc037d34949528b43f62b8e6c81bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee5380aa4eb04532abb6755fb4b0f69e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7d076e82cf8462a93a7565645e2d4b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 2 of 5:   0%|          | 0/990 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9cba17b127d4b0da0431292888e949f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0523b32582c04441a37217d0438a70c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6d708eaf3344f1dabc187a0af3494ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0c558bb357f478bb47ed30de2309eb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "562e088f0c974ea1955972490cc253e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 3 of 5:   0%|          | 0/990 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "532d3702021e4e628bb448adb263d157",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9beb8687113a4a5cb638e7f41eae117c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75636d74041840b2b8236a6316599635",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdcff1c77cb94a229ff5bb23fe61ebf8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1610ca52af74ab6835c3860a2a4dce5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 4 of 5:   0%|          | 0/990 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdd3089ad3944800b5b1d2d0a78c2b60",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f5b158b3f56459c87034dc3137844f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1888c5b6926c441e90deaae903fa6ca7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67b42f5beaeb433bbba2c9c2aa85ee30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4950,\n",
       " defaultdict(list,\n",
       "             {'global_step': [990,\n",
       "               1000,\n",
       "               1980,\n",
       "               2000,\n",
       "               2970,\n",
       "               3000,\n",
       "               3960,\n",
       "               4000,\n",
       "               4950],\n",
       "              'train_loss': [0.11155413836240768,\n",
       "               0.09572186321020126,\n",
       "               0.06548457592725754,\n",
       "               0.07548873871564865,\n",
       "               0.041757550090551376,\n",
       "               0.06851186603307724,\n",
       "               0.05627427622675896,\n",
       "               0.07009167969226837,\n",
       "               0.04540044441819191],\n",
       "              'eval_loss': [0.09899854037741368,\n",
       "               0.09712003184143793,\n",
       "               0.09434240154752677,\n",
       "               0.09744996748949317,\n",
       "               0.09225686061704023,\n",
       "               0.09067362305623564,\n",
       "               0.09103432043709538,\n",
       "               0.09171859676611017,\n",
       "               0.09289782085794616],\n",
       "              'precision': [0.9524399306415655,\n",
       "               0.9525327685557051,\n",
       "               0.95359590965277,\n",
       "               0.9547990128131445,\n",
       "               0.9557764479156101,\n",
       "               0.954604804034128,\n",
       "               0.9538989342256967,\n",
       "               0.954158838023019,\n",
       "               0.9544511011728968],\n",
       "              'recall': [0.9523702298455875,\n",
       "               0.9519536593466599,\n",
       "               0.9554156979526123,\n",
       "               0.9539014078956998,\n",
       "               0.9548133595284872,\n",
       "               0.9548358768527536,\n",
       "               0.9547795835420877,\n",
       "               0.954830247521687,\n",
       "               0.955117343406083],\n",
       "              'f1_score': [0.9524050789683312,\n",
       "               0.9522431259044862,\n",
       "               0.9545049364351375,\n",
       "               0.9543499992960027,\n",
       "               0.9552946609856859,\n",
       "               0.9547203264616899,\n",
       "               0.9543390557215443,\n",
       "               0.9544944247018207,\n",
       "               0.9547841060646134]}))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model\n",
    "model.train_model(train_data, eval_data=val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e1fc6afb-792c-4cdd-b7e7-7d5883d42ee8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/simpletransformers/ner/ner_utils.py:190: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  return [\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6e329fb7ffb4b0eaffad5805796f2c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abd7f55e7108499589683b7be7adb2d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/88 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: VV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NN seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AV seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NU seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PS seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CC seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: AJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: CL seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: NG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: FX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PR seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: XX seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PA seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/home/superai023/.conda/envs/newpt/lib/python3.9/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: IJ seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model\n",
    "result, model_outputs, preds_list = model.eval_model(val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b70dab9f-9794-444e-b69c-526a8266ba94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>words</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>สลด</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>ผัว</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>เมีย</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>เฒ่า</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>ถูก</td>\n",
       "      <td>AX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240855</th>\n",
       "      <td>5619</td>\n",
       "      <td>จะ</td>\n",
       "      <td>AX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240856</th>\n",
       "      <td>5619</td>\n",
       "      <td>ขอ</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240857</th>\n",
       "      <td>5619</td>\n",
       "      <td>ไว้</td>\n",
       "      <td>AV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240858</th>\n",
       "      <td>5619</td>\n",
       "      <td>ดูแล</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240859</th>\n",
       "      <td>5619</td>\n",
       "      <td>เอง</td>\n",
       "      <td>AV</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240860 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        sentence_id words labels\n",
       "0                 0   สลด     VV\n",
       "1                 0   ผัว     NN\n",
       "2                 0  เมีย     NN\n",
       "3                 0  เฒ่า     NN\n",
       "4                 0   ถูก     AX\n",
       "...             ...   ...    ...\n",
       "240855         5619    จะ     AX\n",
       "240856         5619    ขอ     VV\n",
       "240857         5619   ไว้     AV\n",
       "240858         5619  ดูแล     VV\n",
       "240859         5619   เอง     AV\n",
       "\n",
       "[240860 rows x 3 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8ae0021f-6219-4022-a529-1cf10d92da87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.09289782085794616,\n",
       " 'precision': 0.9544511011728968,\n",
       " 'recall': 0.955117343406083,\n",
       " 'f1_score': 0.9547841060646134}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "54b38dd8-4a45-4738-a232-318af85f68d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['VV', 'NN', 'NN', 'NN', 'AX', 'NN', 'VV', 'VV', 'PR', 'VV', 'PU', 'VV', 'NN', 'VV', 'PU', 'NN', 'VV']]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>words</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>สลด</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>ผัว</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>เมีย</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>เฒ่า</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>ถูก</td>\n",
       "      <td>AX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>ไฟ</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>ช็อต</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>กอด</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>กัน</td>\n",
       "      <td>AV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>ตาย</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>_</td>\n",
       "      <td>PU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>สังเวย</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>ฝน</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>ตก</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "      <td>PU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>น้ำ</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>ท่วม</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sentence_id   words labels\n",
       "0             0     สลด     VV\n",
       "1             0     ผัว     NN\n",
       "2             0    เมีย     NN\n",
       "3             0    เฒ่า     NN\n",
       "4             0     ถูก     AX\n",
       "5             0      ไฟ     NN\n",
       "6             0    ช็อต     VV\n",
       "7             0     กอด     VV\n",
       "8             0     กัน     AV\n",
       "9             0     ตาย     VV\n",
       "10            0       _     PU\n",
       "11            0  สังเวย     VV\n",
       "12            0      ฝน     NN\n",
       "13            0      ตก     VV\n",
       "14            0       -     PU\n",
       "15            0     น้ำ     NN\n",
       "16            0    ท่วม     VV"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(preds_list[:1])\n",
    "val_data.loc[val_data['sentence_id'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8043e946-c19d-4529-bcb5-d8d6362d9421",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>words</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>จีน</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "      <td>PU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>อินเดีย</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>เสี่ยง</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>สูญเสีย</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207273</th>\n",
       "      <td>5249</td>\n",
       "      <td>ครหา</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207274</th>\n",
       "      <td>5249</td>\n",
       "      <td>เกี่ยวกับ</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207275</th>\n",
       "      <td>5249</td>\n",
       "      <td>ความ</td>\n",
       "      <td>FX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207276</th>\n",
       "      <td>5249</td>\n",
       "      <td>ไม่</td>\n",
       "      <td>NG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207277</th>\n",
       "      <td>5249</td>\n",
       "      <td>ถูกต้อง</td>\n",
       "      <td>VV</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>207278 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        sentence_id      words labels\n",
       "0                 0        จีน     NN\n",
       "1                 0          -     PU\n",
       "2                 0    อินเดีย     NN\n",
       "3                 0     เสี่ยง     VV\n",
       "4                 0    สูญเสีย     VV\n",
       "...             ...        ...    ...\n",
       "207273         5249       ครหา     VV\n",
       "207274         5249  เกี่ยวกับ     VV\n",
       "207275         5249       ความ     FX\n",
       "207276         5249        ไม่     NG\n",
       "207277         5249    ถูกต้อง     VV\n",
       "\n",
       "[207278 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1b5ba3fa-2840-4fa6-9185-a241a6622e2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>fname</th>\n",
       "      <th>tokens</th>\n",
       "      <th>pos_tags</th>\n",
       "      <th>ner_tags</th>\n",
       "      <th>clause_tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>T11046.txt</td>\n",
       "      <td>[จีน, -, อินเดีย, เสี่ยง, สูญเสีย, จาก, ภัย, ธ...</td>\n",
       "      <td>[0, 2, 0, 1, 1, 4, 0, 0, 1, 6]</td>\n",
       "      <td>[4, 0, 4, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[1, 2, 2, 2, 2, 2, 2, 2, 2, 3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>T11046.txt</td>\n",
       "      <td>[เจนีวา, _, -, _, จีน, _, อินเดีย, _, และ, ประ...</td>\n",
       "      <td>[0, 2, 2, 2, 0, 2, 0, 2, 3, 0, 3, 1, 0, 0, 1, ...</td>\n",
       "      <td>[4, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>T11046.txt</td>\n",
       "      <td>[ใน, รายงาน, ความ, เสี่ยง, จาก, ภัย, ธรรมชาติ,...</td>\n",
       "      <td>[4, 1, 7, 1, 4, 0, 0, 2, 3, 1, 4, 0, 4, 0, 2, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 3, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>T11046.txt</td>\n",
       "      <td>[ประชากร, กลุ่ม, เสี่ยง, ส่วนใหญ่, _, อาศัย, อ...</td>\n",
       "      <td>[0, 10, 1, 0, 2, 1, 1, 4, 4, 0, 0, 2, 1, 3, 2,...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>T11058.txt</td>\n",
       "      <td>[เกิด, เหตุ, พายุ, พัด, ถล่ม, จีน, _, ทำให้, ม...</td>\n",
       "      <td>[1, 0, 0, 1, 1, 0, 2, 3, 1, 7, 1, 7, 1, 2, 8, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6, ...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 2, 2, 2, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5245</th>\n",
       "      <td>15</td>\n",
       "      <td>T13476.txt</td>\n",
       "      <td>[โดย, นาย, ไพโรจน์, _, กล่าว, ว่า, _, การ, แต่...</td>\n",
       "      <td>[3, 0, 0, 2, 1, 3, 2, 7, 1, 10, 9, 2, 1, 7, 12...</td>\n",
       "      <td>[0, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 2, 2, 2, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5246</th>\n",
       "      <td>16</td>\n",
       "      <td>T13476.txt</td>\n",
       "      <td>[และ, ที่, สำคัญ, ที่สุด, คือ, _, นาย, มานิต, ...</td>\n",
       "      <td>[3, 3, 1, 6, 1, 2, 0, 0, 1, 0, 9, 3, 1, 7, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 10, 8, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 1, 2, 2, 2, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5247</th>\n",
       "      <td>17</td>\n",
       "      <td>T13476.txt</td>\n",
       "      <td>[**, _, เชิญ, \", ชวรัตน์, \", _, ชี้แจง, _, 25,...</td>\n",
       "      <td>[2, 2, 1, 2, 0, 2, 2, 1, 2, 8, 2, 0, 9]</td>\n",
       "      <td>[0, 0, 0, 0, 8, 0, 0, 0, 0, 3, 13, 23, 0]</td>\n",
       "      <td>[1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5248</th>\n",
       "      <td>18</td>\n",
       "      <td>T13476.txt</td>\n",
       "      <td>[นาย, ไพโรจน์, _, กล่าว, ด้วย, ว่า, _, คณะ, กร...</td>\n",
       "      <td>[0, 0, 2, 1, 6, 3, 2, 0, 0, 5, 1, 7, 3, 1, 4, ...</td>\n",
       "      <td>[10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 3, 1, 2, 2, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5249</th>\n",
       "      <td>19</td>\n",
       "      <td>T13476.txt</td>\n",
       "      <td>[ด้าน, นาย, ฐนโรจน์, _, กล่าว, ว่า, _, ตน, ใน,...</td>\n",
       "      <td>[0, 0, 0, 2, 1, 3, 2, 11, 4, 0, 0, 4, 0, 12, 1...</td>\n",
       "      <td>[0, 10, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0,...</td>\n",
       "      <td>[1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 2, 2, 2, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5250 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id       fname                                             tokens  \\\n",
       "0      0  T11046.txt  [จีน, -, อินเดีย, เสี่ยง, สูญเสีย, จาก, ภัย, ธ...   \n",
       "1      1  T11046.txt  [เจนีวา, _, -, _, จีน, _, อินเดีย, _, และ, ประ...   \n",
       "2      2  T11046.txt  [ใน, รายงาน, ความ, เสี่ยง, จาก, ภัย, ธรรมชาติ,...   \n",
       "3      3  T11046.txt  [ประชากร, กลุ่ม, เสี่ยง, ส่วนใหญ่, _, อาศัย, อ...   \n",
       "4      0  T11058.txt  [เกิด, เหตุ, พายุ, พัด, ถล่ม, จีน, _, ทำให้, ม...   \n",
       "...   ..         ...                                                ...   \n",
       "5245  15  T13476.txt  [โดย, นาย, ไพโรจน์, _, กล่าว, ว่า, _, การ, แต่...   \n",
       "5246  16  T13476.txt  [และ, ที่, สำคัญ, ที่สุด, คือ, _, นาย, มานิต, ...   \n",
       "5247  17  T13476.txt  [**, _, เชิญ, \", ชวรัตน์, \", _, ชี้แจง, _, 25,...   \n",
       "5248  18  T13476.txt  [นาย, ไพโรจน์, _, กล่าว, ด้วย, ว่า, _, คณะ, กร...   \n",
       "5249  19  T13476.txt  [ด้าน, นาย, ฐนโรจน์, _, กล่าว, ว่า, _, ตน, ใน,...   \n",
       "\n",
       "                                               pos_tags  \\\n",
       "0                        [0, 2, 0, 1, 1, 4, 0, 0, 1, 6]   \n",
       "1     [0, 2, 2, 2, 0, 2, 0, 2, 3, 0, 3, 1, 0, 0, 1, ...   \n",
       "2     [4, 1, 7, 1, 4, 0, 0, 2, 3, 1, 4, 0, 4, 0, 2, ...   \n",
       "3     [0, 10, 1, 0, 2, 1, 1, 4, 4, 0, 0, 2, 1, 3, 2,...   \n",
       "4     [1, 0, 0, 1, 1, 0, 2, 3, 1, 7, 1, 7, 1, 2, 8, ...   \n",
       "...                                                 ...   \n",
       "5245  [3, 0, 0, 2, 1, 3, 2, 7, 1, 10, 9, 2, 1, 7, 12...   \n",
       "5246  [3, 3, 1, 6, 1, 2, 0, 0, 1, 0, 9, 3, 1, 7, 1, ...   \n",
       "5247            [2, 2, 1, 2, 0, 2, 2, 1, 2, 8, 2, 0, 9]   \n",
       "5248  [0, 0, 2, 1, 6, 3, 2, 0, 0, 5, 1, 7, 3, 1, 4, ...   \n",
       "5249  [0, 0, 0, 2, 1, 3, 2, 11, 4, 0, 0, 4, 0, 12, 1...   \n",
       "\n",
       "                                               ner_tags  \\\n",
       "0                        [4, 0, 4, 0, 0, 0, 0, 0, 0, 0]   \n",
       "1     [4, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "2     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "3     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "4     [0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6, ...   \n",
       "...                                                 ...   \n",
       "5245  [0, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "5246  [0, 0, 0, 0, 0, 0, 10, 8, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "5247          [0, 0, 0, 0, 8, 0, 0, 0, 0, 3, 13, 23, 0]   \n",
       "5248  [10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "5249  [0, 10, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0,...   \n",
       "\n",
       "                                            clause_tags  \n",
       "0                        [1, 2, 2, 2, 2, 2, 2, 2, 2, 3]  \n",
       "1     [1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...  \n",
       "2     [1, 2, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 3, 0, ...  \n",
       "3     [1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 0, ...  \n",
       "4     [1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 2, 2, 2, ...  \n",
       "...                                                 ...  \n",
       "5245  [1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 2, 2, 2, ...  \n",
       "5246  [1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 1, 2, 2, 2, ...  \n",
       "5247            [1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3]  \n",
       "5248  [1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 3, 1, 2, 2, ...  \n",
       "5249  [1, 2, 2, 2, 2, 3, 0, 1, 2, 2, 2, 2, 2, 2, 2, ...  \n",
       "\n",
       "[5250 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8bb21d64-a3aa-4d1b-b306-2bfe23253a06",
   "metadata": {},
   "outputs": [],
   "source": [
    "# submission_text = open(\"pos_test.txt\", \"r\")\n",
    "texts_test_raw = []\n",
    "f = open('/project/lt900038-ai23tn/kaggle/nlp_pos_tagging/pos_test.txt', \"r\")\n",
    "file_data = f.readlines()\n",
    "for line in file_data:\n",
    "    line = line.strip().replace(u'\\xa0', u'')\n",
    "    texts_test_raw.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "22cf10aa-6a10-4c13-8cf4-d6e2decfed0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts_test_raw = texts_test_raw[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c7de8e42-d74e-47dd-9fcc-0ce75bbb738e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# n_sub = len(submission_text.split(\"\\n\"))\n",
    "# text_list = []\n",
    "# for i in tqdm(range(n_sub)):\n",
    "#     text = submission_text.split(\"\\n\")[i]\n",
    "#     if text == '':\n",
    "#         text = \"_\"\n",
    "#         text_list.append(text)\n",
    "#     else:\n",
    "#         text_list.append(text)\n",
    "def blank_space(x):\n",
    "    if x == '':\n",
    "        x = '_'\n",
    "    return x\n",
    "\n",
    "for i in range(len(texts_test_raw)):\n",
    "    texts_test_raw[i] = blank_space(texts_test_raw[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e04e6c65-5ee7-46a7-9c5e-37bacc76bc34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103913"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts_test_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c591f22f-81b0-4e2b-833b-5aa7220973fb",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['อย่างไรก็ตาม',\n",
       " 'เครื่อง',\n",
       " 'บิน',\n",
       " 'แอร์บัส',\n",
       " '_',\n",
       " 'เอ',\n",
       " '380',\n",
       " 'ได้',\n",
       " 'ถึง',\n",
       " 'ที่',\n",
       " 'หมาย',\n",
       " 'ที่',\n",
       " 'ท่าอากาศยาน',\n",
       " 'เชียงใหม่',\n",
       " 'ใน',\n",
       " 'เวลา',\n",
       " '_',\n",
       " '14.50',\n",
       " '_',\n",
       " 'น.',\n",
       " 'โดย',\n",
       " 'ใช้',\n",
       " 'เวลา',\n",
       " 'บิน',\n",
       " 'ประมาณ',\n",
       " '1',\n",
       " 'ชั่วโมง',\n",
       " '_',\n",
       " 'มูลนิธิสืบฯ',\n",
       " 'พอใจ',\n",
       " 'ฟื้น',\n",
       " 'ป่าห้วยขาแข้ง',\n",
       " 'มี',\n",
       " 'เสือโคร่ง',\n",
       " 'มาก',\n",
       " 'อันดับ',\n",
       " '2',\n",
       " 'โลก',\n",
       " '_',\n",
       " '1',\n",
       " '_',\n",
       " 'กันยายน',\n",
       " '_',\n",
       " '2550',\n",
       " '17.31',\n",
       " 'น.',\n",
       " '_',\n",
       " 'ประธาน',\n",
       " 'มูลนิธิ',\n",
       " 'สืบ',\n",
       " 'นาคะเสถียร',\n",
       " 'พอใจ',\n",
       " 'ป่า',\n",
       " 'ห้วยขาแข้ง',\n",
       " 'ฟื้นตัว',\n",
       " 'พบ',\n",
       " 'เสือโคร่ง',\n",
       " '2-3',\n",
       " 'ตัว',\n",
       " 'ต่อ',\n",
       " '100',\n",
       " 'ตาราง',\n",
       " 'กิโลเมตร',\n",
       " 'มาก',\n",
       " 'อันดับ',\n",
       " 'ที่',\n",
       " '_',\n",
       " '2',\n",
       " 'ของ',\n",
       " 'โลก',\n",
       " 'ตั้ง',\n",
       " 'เป้า',\n",
       " '5',\n",
       " 'ปี',\n",
       " 'ขึ้น',\n",
       " 'อันดับ',\n",
       " '1',\n",
       " 'แสดง',\n",
       " 'ถึง',\n",
       " 'ยัง',\n",
       " 'มี',\n",
       " 'สัตว์',\n",
       " 'และ',\n",
       " 'พืช',\n",
       " 'ชนิด',\n",
       " 'อื่น',\n",
       " 'ๆ',\n",
       " 'อุดมสมบูรณ์',\n",
       " '_',\n",
       " 'วอน',\n",
       " 'รัฐ',\n",
       " 'เข้าใจ',\n",
       " 'ปรับ',\n",
       " 'นโยบาย',\n",
       " 'จับตา',\n",
       " 'ค้าน',\n",
       " 'กฎหมาย',\n",
       " 'เปิด',\n",
       " 'ป่า',\n",
       " '_',\n",
       " '(',\n",
       " '1',\n",
       " 'ก.ย.',\n",
       " ')',\n",
       " 'นาง',\n",
       " 'รตยา',\n",
       " '_',\n",
       " 'จันทรเทียร',\n",
       " 'ประธาน',\n",
       " 'มูลนิธิ',\n",
       " 'สืบ',\n",
       " 'นาคะเสถียร',\n",
       " 'ให้',\n",
       " 'สัมภาษณ์',\n",
       " 'ถึง',\n",
       " 'สถานการณ์',\n",
       " 'ป่า',\n",
       " 'และ',\n",
       " 'สัตว์',\n",
       " 'ป่า',\n",
       " 'ใน',\n",
       " 'ป่าห้วยขาแข้ง',\n",
       " 'จ.',\n",
       " 'อุทัยธานี',\n",
       " 'ว่า',\n",
       " 'จาก',\n",
       " 'การ',\n",
       " 'วิจัย',\n",
       " 'ทาง',\n",
       " 'วิชาการ',\n",
       " '_',\n",
       " 'ล่าสุด',\n",
       " 'พบ',\n",
       " 'ว่า',\n",
       " 'ผืน',\n",
       " 'ป่า',\n",
       " 'ห้วยขาแข้ง',\n",
       " 'ซึ่ง',\n",
       " 'เป็น',\n",
       " 'ผืน',\n",
       " 'ป่า',\n",
       " 'ขนาด',\n",
       " 'ใหญ่',\n",
       " 'ด้าน',\n",
       " 'ตะวันตก',\n",
       " 'เป็น',\n",
       " 'มรดก',\n",
       " 'ทาง',\n",
       " 'ธรรมชาติ',\n",
       " 'ของ',\n",
       " 'โลก',\n",
       " 'พื้นที่',\n",
       " 'แรก',\n",
       " 'ของ',\n",
       " 'ไทย',\n",
       " 'ตั้งแต่',\n",
       " 'ปี',\n",
       " '2534',\n",
       " '_',\n",
       " 'ขณะ',\n",
       " 'นี้',\n",
       " 'มี',\n",
       " 'เสือโคร่ง',\n",
       " 'อาศัย',\n",
       " 'อยู่',\n",
       " '2-3',\n",
       " 'ตัว',\n",
       " 'ต่อ',\n",
       " 'พื้นที่',\n",
       " '100',\n",
       " 'ตาราง',\n",
       " 'กิโลเมตร',\n",
       " '_',\n",
       " 'หรือ',\n",
       " 'รวม',\n",
       " 'มี',\n",
       " 'เสือโคร่ง',\n",
       " 'ประมาณ',\n",
       " '80',\n",
       " '-',\n",
       " '100',\n",
       " 'ตัว',\n",
       " 'ถือ',\n",
       " 'เป็น',\n",
       " 'อันดับ',\n",
       " '2',\n",
       " 'ของ',\n",
       " 'โลก',\n",
       " 'รอง',\n",
       " 'จาก',\n",
       " 'ผืน',\n",
       " 'ป่า',\n",
       " 'ไซบีเรีย',\n",
       " '_',\n",
       " 'แต่',\n",
       " 'ทาง',\n",
       " 'เจ้าหน้าที่',\n",
       " 'ผู้',\n",
       " 'ดูแล',\n",
       " 'ป่า',\n",
       " 'ได้',\n",
       " 'ตั้ง',\n",
       " 'เป้าหมาย',\n",
       " 'ว่า',\n",
       " 'ภาย',\n",
       " 'ใน',\n",
       " '5',\n",
       " 'ปี',\n",
       " 'จะ',\n",
       " 'ต้อง',\n",
       " 'เพิ่ม',\n",
       " 'ปริมาณ',\n",
       " 'เสือโคร่ง',\n",
       " 'เป็น',\n",
       " '5',\n",
       " 'ตัว',\n",
       " 'ต่อ',\n",
       " '100',\n",
       " 'ตาราง',\n",
       " 'กิโลเมตร',\n",
       " 'หรือ',\n",
       " 'รวม',\n",
       " '200',\n",
       " 'ตัว',\n",
       " 'ซึ่ง',\n",
       " 'จะ',\n",
       " 'ทำ',\n",
       " 'ให้',\n",
       " 'เลื่อน',\n",
       " 'ขึ้น',\n",
       " 'เป็น',\n",
       " 'อันดับ',\n",
       " 'ที่',\n",
       " '_',\n",
       " '1',\n",
       " 'ของ',\n",
       " 'โลก',\n",
       " '_',\n",
       " 'ทั้งนี้',\n",
       " 'ถือ',\n",
       " 'เป็น',\n",
       " 'แนวโน้ม',\n",
       " 'ที่',\n",
       " 'น่า',\n",
       " 'พอใจ',\n",
       " 'อย่าง',\n",
       " 'ยิ่ง',\n",
       " 'โดย',\n",
       " 'เตรียม',\n",
       " 'จะ',\n",
       " 'ขยาย',\n",
       " 'โครงการ',\n",
       " 'ศึกษา',\n",
       " 'ไป',\n",
       " 'ถึง',\n",
       " 'ซึ่ง',\n",
       " 'เป็น',\n",
       " 'ผืน',\n",
       " 'ป่า',\n",
       " 'ด้าน',\n",
       " 'ตะวันตก',\n",
       " 'ที่',\n",
       " 'อยู่',\n",
       " 'ติดต่อ',\n",
       " 'กัน',\n",
       " 'ด้วย',\n",
       " '_',\n",
       " 'สำหรับ',\n",
       " 'ปัจจัย',\n",
       " 'ความ',\n",
       " 'สำเร็จ',\n",
       " 'คือ',\n",
       " 'เจ้าหน้าที่',\n",
       " 'เขต',\n",
       " 'รักษา',\n",
       " 'พันธุ์',\n",
       " 'สัตว์',\n",
       " 'ป่า',\n",
       " 'ที่',\n",
       " 'ทำ',\n",
       " 'งาน',\n",
       " 'กัน',\n",
       " 'อย่าง',\n",
       " 'เข้มแข็ง',\n",
       " 'เจ้าหน้าที่',\n",
       " 'รัฐ',\n",
       " 'นัก',\n",
       " 'วิจัย',\n",
       " 'รวม',\n",
       " 'ถึง',\n",
       " 'ชุมชน',\n",
       " 'ใน',\n",
       " 'พื้นที่',\n",
       " 'ชาย',\n",
       " 'จำนวน',\n",
       " 'เสือโคร่ง',\n",
       " 'ใน',\n",
       " 'ป่า',\n",
       " 'มี',\n",
       " 'ความหมาย',\n",
       " 'ถึง',\n",
       " 'สิ่ง',\n",
       " 'มี',\n",
       " 'ชีวิต',\n",
       " 'อื่น',\n",
       " 'ๆ',\n",
       " 'เพราะ',\n",
       " 'อาหาร',\n",
       " 'ของ',\n",
       " 'เสือโคร่ง',\n",
       " 'คือ',\n",
       " 'สัตว์',\n",
       " 'กีบ',\n",
       " 'เช่น',\n",
       " 'เก้ง',\n",
       " 'กวาง',\n",
       " 'วัว',\n",
       " 'ลูก',\n",
       " 'กระทิง',\n",
       " '_',\n",
       " 'เมื่อ',\n",
       " 'เสือโคร่ง',\n",
       " 'อยู่',\n",
       " 'ได้',\n",
       " 'ก็',\n",
       " 'แสดง',\n",
       " 'ว่า',\n",
       " 'ต้อง',\n",
       " 'มี',\n",
       " 'สัตว์',\n",
       " 'กีบ',\n",
       " 'อาศัย',\n",
       " 'อยู่',\n",
       " 'เพียงพอ',\n",
       " 'และ',\n",
       " 'ยัง',\n",
       " 'หมายถึง',\n",
       " 'พืช',\n",
       " 'ที่',\n",
       " 'เป็น',\n",
       " 'อาหาร',\n",
       " 'ของ',\n",
       " 'สัตว์',\n",
       " 'กีบ',\n",
       " 'เหล่า',\n",
       " 'นี้',\n",
       " 'ยัง',\n",
       " 'มี',\n",
       " 'อย่าง',\n",
       " 'อุดมสมบูรณ์',\n",
       " 'เพียงพอ',\n",
       " 'เช่น',\n",
       " 'กัน',\n",
       " 'ก็',\n",
       " 'แสดง',\n",
       " 'ว่า',\n",
       " 'ป่า',\n",
       " 'ยัง',\n",
       " 'สมบูรณ์',\n",
       " '_',\n",
       " 'แต่',\n",
       " 'เรา',\n",
       " 'ก็',\n",
       " 'มี',\n",
       " 'เป้าหมาย',\n",
       " 'จะ',\n",
       " 'เพิ่ม',\n",
       " 'ความ',\n",
       " 'อุดมสมบูรณ์',\n",
       " 'ให้',\n",
       " 'ไป',\n",
       " 'ถึง',\n",
       " 'เสือโคร่ง',\n",
       " '5',\n",
       " 'ตัว',\n",
       " 'ต่อ',\n",
       " '100',\n",
       " 'ตาราง',\n",
       " 'ภาย',\n",
       " 'ใน',\n",
       " '5',\n",
       " 'ปี',\n",
       " 'นาง',\n",
       " 'รตยา',\n",
       " 'กล่าว',\n",
       " '_',\n",
       " 'นาง',\n",
       " 'รตยา',\n",
       " 'ซึ่ง',\n",
       " 'อยู่',\n",
       " 'ระหว่าง',\n",
       " 'ร่วม',\n",
       " 'งาน',\n",
       " 'วัน',\n",
       " 'รำลึก',\n",
       " '17',\n",
       " 'ปี',\n",
       " 'สืบ',\n",
       " '_',\n",
       " 'นาคะเสถียร',\n",
       " 'ณ',\n",
       " 'อาคาร',\n",
       " 'อนุสรณ์สถาน',\n",
       " '_',\n",
       " 'สืบ',\n",
       " '_',\n",
       " 'นาคะเสถียร',\n",
       " 'เขต',\n",
       " 'รักษา',\n",
       " 'พันธุ์',\n",
       " 'สัตว์',\n",
       " 'ป่า',\n",
       " 'ห้วยขาแข้ง',\n",
       " '_',\n",
       " 'อำเภอ',\n",
       " 'ลานสัก',\n",
       " 'จังหวัด',\n",
       " 'อุทัยธานี',\n",
       " 'กล่าว',\n",
       " 'ด้วย',\n",
       " 'ว่า',\n",
       " 'แม้',\n",
       " 'แนวโน้ม',\n",
       " 'การ',\n",
       " 'อนุรักษ์',\n",
       " 'ฟื้นฟู',\n",
       " 'ป่า',\n",
       " 'จะ',\n",
       " 'ไป',\n",
       " 'ใน',\n",
       " 'ทาง',\n",
       " 'ที่',\n",
       " 'ดี',\n",
       " 'ขึ้น',\n",
       " 'แต่',\n",
       " 'ก็',\n",
       " 'ยัง',\n",
       " 'ต้อง',\n",
       " 'มี',\n",
       " 'การ',\n",
       " 'ทำ',\n",
       " 'ความ',\n",
       " 'เข้าใจ',\n",
       " 'กับ',\n",
       " 'หลาย',\n",
       " 'ฝ่าย',\n",
       " 'อีก',\n",
       " 'มาก',\n",
       " '_',\n",
       " 'โดย',\n",
       " 'เฉพาะ',\n",
       " 'ภาค',\n",
       " 'รัฐ',\n",
       " 'มัก',\n",
       " 'ดำเนิน',\n",
       " 'โครงการ',\n",
       " 'ก้ำกึ่ง',\n",
       " 'ด้าน',\n",
       " 'หนึ่ง',\n",
       " 'อนุรักษ์',\n",
       " 'แต่',\n",
       " 'อีก',\n",
       " 'ด้าน',\n",
       " 'หนึ่ง',\n",
       " 'ก็',\n",
       " 'มี',\n",
       " 'ส่วน',\n",
       " 'ทำ',\n",
       " 'ให้',\n",
       " 'ผืน',\n",
       " 'ป่า',\n",
       " 'หมด',\n",
       " 'ไป',\n",
       " 'เช่น',\n",
       " 'โครงการ',\n",
       " 'อุโมงค์',\n",
       " 'ผัน',\n",
       " 'น้ำ',\n",
       " 'จาก',\n",
       " 'เขื่อน',\n",
       " 'ศรีนครินทร์',\n",
       " 'ผ่าน',\n",
       " 'พื้นที่',\n",
       " 'ป่า',\n",
       " 'มา',\n",
       " 'ยัง',\n",
       " 'จังหวัด',\n",
       " 'สุพรรณบุรี',\n",
       " 'โครงการ',\n",
       " 'สร้าง',\n",
       " 'เขื่อน',\n",
       " 'แม่วงก์',\n",
       " 'จังหวัด',\n",
       " 'นครสวรรค์',\n",
       " 'โครงการ',\n",
       " 'ส่งเสริม',\n",
       " 'การ',\n",
       " 'ท่องเที่ยว',\n",
       " 'บาง',\n",
       " 'โครงการ',\n",
       " 'โครงการ',\n",
       " 'ส่งเสริม',\n",
       " 'การ',\n",
       " 'ปลูก',\n",
       " 'พืช',\n",
       " 'เชิง',\n",
       " 'เดี่ยว',\n",
       " 'โครงการ',\n",
       " 'ตัด',\n",
       " 'ถนน',\n",
       " 'ผ่าน',\n",
       " 'ป่า',\n",
       " 'การ',\n",
       " 'ลักลอบ',\n",
       " 'ตัด',\n",
       " 'ไม้',\n",
       " 'ใน',\n",
       " 'ปัจจุบัน',\n",
       " 'เหลือ',\n",
       " 'ไม่',\n",
       " 'มาก',\n",
       " 'แล้ว',\n",
       " 'เนื่องจาก',\n",
       " 'ไม้',\n",
       " 'เป็น',\n",
       " 'ของ',\n",
       " 'ใหญ่',\n",
       " '_',\n",
       " 'ถึงอย่างไร',\n",
       " 'ก็',\n",
       " 'คง',\n",
       " 'จะ',\n",
       " 'ไม่',\n",
       " 'หมด',\n",
       " 'ไป',\n",
       " 'เท่า',\n",
       " 'ที่',\n",
       " 'คน',\n",
       " 'ยัง',\n",
       " 'มี',\n",
       " 'กิเลส',\n",
       " 'แต่',\n",
       " 'ใน',\n",
       " 'ปัจจุบัน',\n",
       " 'สิ่ง',\n",
       " 'ที่',\n",
       " 'ส่ง',\n",
       " 'ผล',\n",
       " 'กระทบ',\n",
       " 'ต่อ',\n",
       " 'พื้นที่',\n",
       " 'ป่า',\n",
       " 'เป็น',\n",
       " 'วง',\n",
       " 'กว้าง',\n",
       " 'กว่า',\n",
       " 'คือ',\n",
       " 'โครงการ',\n",
       " 'และ',\n",
       " 'นโยบาย',\n",
       " 'ของ',\n",
       " 'รัฐบาล',\n",
       " 'ที่',\n",
       " 'จะ',\n",
       " 'เป็น',\n",
       " 'โครงการ',\n",
       " 'ขนาด',\n",
       " 'ใหญ่',\n",
       " 'เมื่อ',\n",
       " 'ดำเนิน',\n",
       " 'การ',\n",
       " 'จะ',\n",
       " 'กิน',\n",
       " 'อาณาเขต',\n",
       " 'พื้นที่',\n",
       " 'กว้างขวาง',\n",
       " 'ทั้ง',\n",
       " 'นโยบาย',\n",
       " 'ทาง',\n",
       " 'ตรง',\n",
       " 'และ',\n",
       " 'การ',\n",
       " 'ส่งเสริม',\n",
       " 'ทาง',\n",
       " 'อ้อม',\n",
       " '_',\n",
       " 'ส่วน',\n",
       " 'การ',\n",
       " 'บุกรุก',\n",
       " 'พื้นที่',\n",
       " 'ป่า',\n",
       " 'ของ',\n",
       " 'ชาว',\n",
       " 'บ้าน',\n",
       " 'ชุมชน',\n",
       " 'นั้น',\n",
       " 'ส่วน',\n",
       " 'ใหญ่',\n",
       " 'จะ',\n",
       " 'รู้',\n",
       " 'ว่า',\n",
       " 'ขอบเขต',\n",
       " 'อย่างไร',\n",
       " '_',\n",
       " 'ประธาน',\n",
       " 'มูลนิธิ',\n",
       " 'สืบ',\n",
       " 'ฯ',\n",
       " '_',\n",
       " 'กล่าว',\n",
       " 'และ',\n",
       " 'ว่า',\n",
       " 'สำหรับ',\n",
       " 'ภาค',\n",
       " 'ประชาชน',\n",
       " 'เชื่อ',\n",
       " 'ว่า',\n",
       " 'ส่วน',\n",
       " 'ใหญ่',\n",
       " 'เข้าใจ',\n",
       " 'แล้ว',\n",
       " 'ว่า',\n",
       " 'การ',\n",
       " 'รักษา',\n",
       " 'ผืน',\n",
       " 'ป่า',\n",
       " 'เป็น',\n",
       " 'ความ',\n",
       " 'จำเป็น',\n",
       " 'และ',\n",
       " 'ป่า',\n",
       " 'กำลัง',\n",
       " 'จะ',\n",
       " 'หมด',\n",
       " 'ลง',\n",
       " 'ทั้งนี้',\n",
       " 'ยัง',\n",
       " 'ต้องการ',\n",
       " 'ความ',\n",
       " 'ร่วมมือ',\n",
       " 'จาก',\n",
       " 'สื่อ',\n",
       " 'มวล',\n",
       " 'ชน',\n",
       " 'และ',\n",
       " 'ทุก',\n",
       " 'ภาค',\n",
       " 'ส่วน',\n",
       " 'ด้วย',\n",
       " '_',\n",
       " 'ประธาน',\n",
       " 'มูลนิธิ',\n",
       " 'สืบ',\n",
       " 'ฯ',\n",
       " '_',\n",
       " 'กล่าว',\n",
       " 'ว่า',\n",
       " 'ใน',\n",
       " 'ปี',\n",
       " 'ต่อไป',\n",
       " 'ทาง',\n",
       " 'มูลนิธิ',\n",
       " 'ฯ',\n",
       " '_',\n",
       " 'จะ',\n",
       " 'เฝ้า',\n",
       " 'ระวัง',\n",
       " 'จับตา',\n",
       " 'กฎหมาย',\n",
       " 'ที่',\n",
       " 'อาจ',\n",
       " 'ส่ง',\n",
       " 'ผล',\n",
       " 'กระทบ',\n",
       " 'ต่อ',\n",
       " 'ผืน',\n",
       " 'ป่า',\n",
       " 'และ',\n",
       " 'สัตว์',\n",
       " 'ป่า',\n",
       " 'เปิด',\n",
       " 'ป่า',\n",
       " 'และ',\n",
       " 'จำหน่าย',\n",
       " 'สัตว์',\n",
       " 'ป่า',\n",
       " 'โครงการ',\n",
       " 'ขนาด',\n",
       " 'ใหญ่',\n",
       " 'ของ',\n",
       " 'ภาค',\n",
       " 'รัฐ',\n",
       " 'และ',\n",
       " 'การ',\n",
       " 'เสริมสร้าง',\n",
       " 'ความ',\n",
       " 'เข้าใจ',\n",
       " 'ใน',\n",
       " 'ชุมชน',\n",
       " 'กับ',\n",
       " 'ป่า',\n",
       " 'ซึ่ง',\n",
       " 'เป็น',\n",
       " 'กลไก',\n",
       " 'สำคัญ',\n",
       " 'ใน',\n",
       " 'การ',\n",
       " 'รักษา',\n",
       " 'ป่า',\n",
       " '_',\n",
       " 'ผู้',\n",
       " 'สื่อ',\n",
       " 'ข่าว',\n",
       " 'รายงาน',\n",
       " 'ว่า',\n",
       " 'สำหรับ',\n",
       " 'ใน',\n",
       " 'วัน',\n",
       " 'นี้',\n",
       " '(',\n",
       " '1',\n",
       " '_',\n",
       " 'ก.ย.',\n",
       " ')',\n",
       " 'ใน',\n",
       " 'งาน',\n",
       " 'วัน',\n",
       " 'รำลึก',\n",
       " '17',\n",
       " 'ปี',\n",
       " 'สืบ',\n",
       " '_',\n",
       " 'นาคะเสถียร',\n",
       " 'ณ',\n",
       " '_',\n",
       " 'อาคาร',\n",
       " 'อนุสรณ์สถาน',\n",
       " '_',\n",
       " 'สืบ',\n",
       " '_',\n",
       " 'นาคะเสถียร',\n",
       " 'เขต',\n",
       " 'รักษา',\n",
       " 'พันธุ์',\n",
       " 'สัตว์',\n",
       " 'ป่า',\n",
       " 'ห้วยขาแข้ง',\n",
       " 'อำเภอ',\n",
       " 'ลานสัก',\n",
       " 'จังหวัด',\n",
       " 'อุทัยธานี',\n",
       " 'มี',\n",
       " 'กิจกรรม',\n",
       " 'ตั้งแต่',\n",
       " 'เวลา',\n",
       " '_',\n",
       " '08.00',\n",
       " '_',\n",
       " 'น.',\n",
       " 'ประกอบ',\n",
       " 'ด้วย',\n",
       " '_',\n",
       " 'พิธี',\n",
       " 'สงฆ์',\n",
       " 'ทำ',\n",
       " 'บุญ',\n",
       " 'ตักบาตร',\n",
       " 'อุทิศ',\n",
       " 'ส่วน',\n",
       " 'กุศล',\n",
       " 'ให้',\n",
       " 'กับ',\n",
       " 'สืบ',\n",
       " '_',\n",
       " 'นาคะเสถียร',\n",
       " 'และ',\n",
       " 'เจ้าหน้าที่',\n",
       " 'ผู้',\n",
       " 'เสียสละ',\n",
       " 'ชีวิต',\n",
       " 'เพื่อ',\n",
       " 'ปกป้อง',\n",
       " 'ทรัพยากร',\n",
       " 'ธรรมชาติ',\n",
       " 'ป่าไม้',\n",
       " 'สัตว์',\n",
       " 'ป่า',\n",
       " '_',\n",
       " 'จาก',\n",
       " 'นั้น',\n",
       " 'เป็น',\n",
       " 'พิธี',\n",
       " 'รำลึก',\n",
       " '17',\n",
       " 'ปี',\n",
       " 'สืบ',\n",
       " '_',\n",
       " 'นาคะเสถียร',\n",
       " 'ซึ่ง',\n",
       " 'จะ',\n",
       " 'มี',\n",
       " 'การ',\n",
       " 'อ่าน',\n",
       " 'ชีวประวัติ',\n",
       " 'สืบ',\n",
       " '_',\n",
       " 'นาคะเสถียร',\n",
       " 'กล่าว',\n",
       " 'สดุดี',\n",
       " 'และ',\n",
       " 'ยืน',\n",
       " 'สงบ',\n",
       " 'นิ่ง',\n",
       " 'เพื่อ',\n",
       " 'ไว้',\n",
       " 'อาลัย',\n",
       " '_',\n",
       " 'ระทึก',\n",
       " '!!',\n",
       " 'ปีก',\n",
       " 'แอร์บัส',\n",
       " 'เฉี่ยว',\n",
       " 'โรง',\n",
       " 'ซ่อม',\n",
       " 'ช่วง',\n",
       " 'ขึ้น',\n",
       " 'บิน',\n",
       " 'สุวรรณภูมิ',\n",
       " '-',\n",
       " 'เชียงใหม่',\n",
       " '_',\n",
       " '1',\n",
       " '_',\n",
       " 'กันยายน',\n",
       " '_',\n",
       " '2550',\n",
       " '17.31',\n",
       " 'น.',\n",
       " '_',\n",
       " 'เครื่อง',\n",
       " 'บิน',\n",
       " 'แอร์บัส',\n",
       " '_',\n",
       " 'เอ',\n",
       " '380',\n",
       " 'บิน',\n",
       " 'สาธิต',\n",
       " 'จาก',\n",
       " 'สุวรรณภูมิ',\n",
       " 'ไป',\n",
       " 'เชียงใหม่',\n",
       " 'แต่',\n",
       " 'ระหว่าง',\n",
       " 'นำ',\n",
       " 'เครื่อง',\n",
       " 'ขึ้น',\n",
       " 'ปีก',\n",
       " 'ด้าน',\n",
       " 'ซ้าย',\n",
       " 'เฉี่ยว',\n",
       " 'ชน',\n",
       " 'อาคาร',\n",
       " 'โรง',\n",
       " 'ซ่อม',\n",
       " 'เครื่อง',\n",
       " 'บิน',\n",
       " 'จน',\n",
       " 'ปีก',\n",
       " 'เครื่อง',\n",
       " 'ได้',\n",
       " 'รับ',\n",
       " 'ความ',\n",
       " 'เสียหาย',\n",
       " '_',\n",
       " 'กัปตัน',\n",
       " 'ยืนยัน',\n",
       " 'เดินทาง',\n",
       " 'ต่อ',\n",
       " 'จน',\n",
       " 'ถึง',\n",
       " '_',\n",
       " 'เผย',\n",
       " 'ไทย',\n",
       " 'สั่ง',\n",
       " 'ซื้อ',\n",
       " '6',\n",
       " 'ลำ',\n",
       " 'กำหนด',\n",
       " 'ส่ง',\n",
       " 'มอบ',\n",
       " 'ปี',\n",
       " '54',\n",
       " '_',\n",
       " '(',\n",
       " '1',\n",
       " 'ก.ย.',\n",
       " ')',\n",
       " 'เวลา',\n",
       " '_',\n",
       " '9.30',\n",
       " 'น.',\n",
       " 'บริษัท',\n",
       " 'แอร์บัส',\n",
       " '_',\n",
       " 'ได้',\n",
       " 'นำ',\n",
       " 'เครื่อง',\n",
       " 'บิน',\n",
       " '_',\n",
       " 'แอร์บัส',\n",
       " '_',\n",
       " 'เอ',\n",
       " '380',\n",
       " 'เครื่อง',\n",
       " 'บิน',\n",
       " 'พาณิชย์',\n",
       " 'ที่',\n",
       " 'ใหญ่',\n",
       " 'ที่สุด',\n",
       " 'ใน',\n",
       " 'โลก',\n",
       " 'ออก',\n",
       " 'บิน',\n",
       " 'สาธิต',\n",
       " 'จาก',\n",
       " 'ท่า',\n",
       " 'อากาศยาน',\n",
       " 'สุวรรณภูมิ',\n",
       " '_',\n",
       " 'โดย',\n",
       " 'เที่ยว',\n",
       " 'บิน',\n",
       " 'ทีจี',\n",
       " '1380',\n",
       " 'มี',\n",
       " 'กำหนด',\n",
       " 'เดินทาง',\n",
       " 'จาก',\n",
       " 'กรุงเทพ',\n",
       " 'ใน',\n",
       " 'เวลา',\n",
       " '_',\n",
       " '09.30',\n",
       " 'น.',\n",
       " 'ถึง',\n",
       " 'เชียงใหม่',\n",
       " 'ใน',\n",
       " 'เวลา',\n",
       " '_',\n",
       " '10.30',\n",
       " '_',\n",
       " 'น.',\n",
       " '_',\n",
       " 'โดย',\n",
       " 'เครื่อง',\n",
       " 'บิน',\n",
       " 'ลำ',\n",
       " 'ดัง',\n",
       " 'กล่าว',\n",
       " 'เดินทาง',\n",
       " 'มา',\n",
       " 'ถึง',\n",
       " 'ประเทศ',\n",
       " 'ไทย',\n",
       " 'วัน',\n",
       " 'ที่',\n",
       " '_',\n",
       " '31',\n",
       " '_',\n",
       " 'สิงหาคม',\n",
       " 'ที่',\n",
       " 'ผ่าน',\n",
       " 'มา',\n",
       " '_',\n",
       " 'และ',\n",
       " 'การ',\n",
       " 'บิน',\n",
       " 'วัน',\n",
       " 'นี้',\n",
       " 'เป็น',\n",
       " 'ส่วน',\n",
       " 'หนึ่ง',\n",
       " 'ของ',\n",
       " 'การ',\n",
       " 'บิน',\n",
       " 'สาธิต',\n",
       " 'ใน',\n",
       " 'เอเชีย',\n",
       " 'ทั้ง',\n",
       " 'เวียดนาม',\n",
       " 'ฮ่องกง',\n",
       " 'เที่ยว',\n",
       " 'บิน',\n",
       " 'ดัง',\n",
       " 'กล่าว',\n",
       " 'มี',\n",
       " 'การ',\n",
       " 'เชิญ',\n",
       " 'แขก',\n",
       " 'ผู้',\n",
       " ...]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts_test_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "53400b09-2ff0-458d-9b3f-20f6cdd7f19c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_df = pd.DataFrame()\n",
    "sub_df[\"text\"] = texts_test_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4306bb52-1849-40ac-bd06-2854e1fd80db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>อย่างไรก็ตาม</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>เครื่อง</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>บิน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>แอร์บัส</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103908</th>\n",
       "      <td>เตรียม</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103909</th>\n",
       "      <td>พร้อม</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103910</th>\n",
       "      <td>รับ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103911</th>\n",
       "      <td>การ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103912</th>\n",
       "      <td>โจมตี</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103913 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                text\n",
       "0       อย่างไรก็ตาม\n",
       "1            เครื่อง\n",
       "2                บิน\n",
       "3            แอร์บัส\n",
       "4                  _\n",
       "...              ...\n",
       "103908        เตรียม\n",
       "103909         พร้อม\n",
       "103910           รับ\n",
       "103911           การ\n",
       "103912         โจมตี\n",
       "\n",
       "[103913 rows x 1 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "49d90fa0-13da-4b67-90d6-477ac039730a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    อย่างไรก็ตาม\n",
       "1         เครื่อง\n",
       "2             บิน\n",
       "3         แอร์บัส\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df[\"text\"].iloc[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "26ad3703-583b-47a7-8a28-856ed0cd463a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93811976f6fb401dbf255265beccca67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/125 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12df290402b04a289699c2807ce27084",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1624 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_test_sub,raw_outputs_test= model.predict(sub_df[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2c5b5183-f931-41a6-96f9-5ffdb006bc42",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[{'อย่างไรก็ตาม': 'CC'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'แอร์บัส': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'เอ': 'NN'}],\n",
       " [{'380': 'NU'}],\n",
       " [{'ได้': 'AX'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'หมาย': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'ท่าอากาศยาน': 'NN'}],\n",
       " [{'เชียงใหม่': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'เวลา': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'14.50': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'น.': 'NN'}],\n",
       " [{'โดย': 'CC'}],\n",
       " [{'ใช้': 'VV'}],\n",
       " [{'เวลา': 'NN'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'ประมาณ': 'AJ'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'ชั่วโมง': 'CL'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'มูลนิธิสืบฯ': 'NN'}],\n",
       " [{'พอใจ': 'VV'}],\n",
       " [{'ฟื้น': 'VV'}],\n",
       " [{'ป่าห้วยขาแข้ง': 'NN'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'มาก': 'VV'}],\n",
       " [{'อันดับ': 'NN'}],\n",
       " [{'2': 'NU'}],\n",
       " [{'โลก': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'กันยายน': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'2550': 'NU'}],\n",
       " [{'17.31': 'NU'}],\n",
       " [{'น.': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ประธาน': 'NN'}],\n",
       " [{'มูลนิธิ': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'พอใจ': 'VV'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ห้วยขาแข้ง': 'NN'}],\n",
       " [{'ฟื้นตัว': 'VV'}],\n",
       " [{'พบ': 'VV'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'2-3': 'NU'}],\n",
       " [{'ตัว': 'NN'}],\n",
       " [{'ต่อ': 'PS'}],\n",
       " [{'100': 'NU'}],\n",
       " [{'ตาราง': 'NN'}],\n",
       " [{'กิโลเมตร': 'CL'}],\n",
       " [{'มาก': 'VV'}],\n",
       " [{'อันดับ': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'2': 'NU'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'โลก': 'NN'}],\n",
       " [{'ตั้ง': 'VV'}],\n",
       " [{'เป้า': 'NN'}],\n",
       " [{'5': 'NU'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'ขึ้น': 'VV'}],\n",
       " [{'อันดับ': 'NN'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'แสดง': 'VV'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'พืช': 'NN'}],\n",
       " [{'ชนิด': 'NN'}],\n",
       " [{'อื่น': 'AJ'}],\n",
       " [{'ๆ': 'NN'}],\n",
       " [{'อุดมสมบูรณ์': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'วอน': 'VV'}],\n",
       " [{'รัฐ': 'NN'}],\n",
       " [{'เข้าใจ': 'VV'}],\n",
       " [{'ปรับ': 'VV'}],\n",
       " [{'นโยบาย': 'NN'}],\n",
       " [{'จับตา': 'VV'}],\n",
       " [{'ค้าน': 'VV'}],\n",
       " [{'กฎหมาย': 'NN'}],\n",
       " [{'เปิด': 'VV'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'(': 'PU'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'ก.ย.': 'NN'}],\n",
       " [{')': 'PU'}],\n",
       " [{'นาง': 'NN'}],\n",
       " [{'รตยา': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'จันทรเทียร': 'NN'}],\n",
       " [{'ประธาน': 'NN'}],\n",
       " [{'มูลนิธิ': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'ให้': 'CC'}],\n",
       " [{'สัมภาษณ์': 'VV'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'สถานการณ์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'ป่าห้วยขาแข้ง': 'NN'}],\n",
       " [{'จ.': 'NN'}],\n",
       " [{'อุทัยธานี': 'NN'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'วิจัย': 'VV'}],\n",
       " [{'ทาง': 'NN'}],\n",
       " [{'วิชาการ': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ล่าสุด': 'AV'}],\n",
       " [{'พบ': 'VV'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'ผืน': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ห้วยขาแข้ง': 'NN'}],\n",
       " [{'ซึ่ง': 'CC'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'ผืน': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ขนาด': 'NN'}],\n",
       " [{'ใหญ่': 'VV'}],\n",
       " [{'ด้าน': 'NN'}],\n",
       " [{'ตะวันตก': 'NN'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'มรดก': 'NN'}],\n",
       " [{'ทาง': 'NN'}],\n",
       " [{'ธรรมชาติ': 'NN'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'โลก': 'NN'}],\n",
       " [{'พื้นที่': 'NN'}],\n",
       " [{'แรก': 'NU'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'ไทย': 'NN'}],\n",
       " [{'ตั้งแต่': 'PS'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'2534': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ขณะ': 'CC'}],\n",
       " [{'นี้': 'AJ'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'อาศัย': 'VV'}],\n",
       " [{'อยู่': 'VV'}],\n",
       " [{'2-3': 'NU'}],\n",
       " [{'ตัว': 'NN'}],\n",
       " [{'ต่อ': 'PS'}],\n",
       " [{'พื้นที่': 'NN'}],\n",
       " [{'100': 'NU'}],\n",
       " [{'ตาราง': 'NN'}],\n",
       " [{'กิโลเมตร': 'CL'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'หรือ': 'CC'}],\n",
       " [{'รวม': 'VV'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'ประมาณ': 'AJ'}],\n",
       " [{'80': 'NU'}],\n",
       " [{'-': 'PU'}],\n",
       " [{'100': 'NU'}],\n",
       " [{'ตัว': 'NN'}],\n",
       " [{'ถือ': 'VV'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'อันดับ': 'NN'}],\n",
       " [{'2': 'NU'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'โลก': 'NN'}],\n",
       " [{'รอง': 'NN'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'ผืน': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ไซบีเรีย': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'แต่': 'CC'}],\n",
       " [{'ทาง': 'NN'}],\n",
       " [{'เจ้าหน้าที่': 'NN'}],\n",
       " [{'ผู้': 'FX'}],\n",
       " [{'ดูแล': 'VV'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ได้': 'AX'}],\n",
       " [{'ตั้ง': 'VV'}],\n",
       " [{'เป้าหมาย': 'NN'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'ภาย': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'5': 'NU'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'ต้อง': 'AX'}],\n",
       " [{'เพิ่ม': 'VV'}],\n",
       " [{'ปริมาณ': 'NN'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'5': 'NU'}],\n",
       " [{'ตัว': 'NN'}],\n",
       " [{'ต่อ': 'PS'}],\n",
       " [{'100': 'NU'}],\n",
       " [{'ตาราง': 'NN'}],\n",
       " [{'กิโลเมตร': 'CL'}],\n",
       " [{'หรือ': 'CC'}],\n",
       " [{'รวม': 'VV'}],\n",
       " [{'200': 'NU'}],\n",
       " [{'ตัว': 'NN'}],\n",
       " [{'ซึ่ง': 'CC'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'ทำ': 'VV'}],\n",
       " [{'ให้': 'CC'}],\n",
       " [{'เลื่อน': 'VV'}],\n",
       " [{'ขึ้น': 'VV'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'อันดับ': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'โลก': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ทั้งนี้': 'CC'}],\n",
       " [{'ถือ': 'VV'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'แนวโน้ม': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'น่า': 'AX'}],\n",
       " [{'พอใจ': 'VV'}],\n",
       " [{'อย่าง': 'FX'}],\n",
       " [{'ยิ่ง': 'AV'}],\n",
       " [{'โดย': 'CC'}],\n",
       " [{'เตรียม': 'VV'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'ขยาย': 'VV'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'ศึกษา': 'VV'}],\n",
       " [{'ไป': 'VV'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'ซึ่ง': 'CC'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'ผืน': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ด้าน': 'NN'}],\n",
       " [{'ตะวันตก': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'อยู่': 'VV'}],\n",
       " [{'ติดต่อ': 'VV'}],\n",
       " [{'กัน': 'PR'}],\n",
       " [{'ด้วย': 'PS'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'สำหรับ': 'PS'}],\n",
       " [{'ปัจจัย': 'NN'}],\n",
       " [{'ความ': 'FX'}],\n",
       " [{'สำเร็จ': 'VV'}],\n",
       " [{'คือ': 'VV'}],\n",
       " [{'เจ้าหน้าที่': 'NN'}],\n",
       " [{'เขต': 'NN'}],\n",
       " [{'รักษา': 'VV'}],\n",
       " [{'พันธุ์': 'NN'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'ทำ': 'VV'}],\n",
       " [{'งาน': 'NN'}],\n",
       " [{'กัน': 'PR'}],\n",
       " [{'อย่าง': 'FX'}],\n",
       " [{'เข้มแข็ง': 'VV'}],\n",
       " [{'เจ้าหน้าที่': 'NN'}],\n",
       " [{'รัฐ': 'NN'}],\n",
       " [{'นัก': 'FX'}],\n",
       " [{'วิจัย': 'VV'}],\n",
       " [{'รวม': 'VV'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'ชุมชน': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'พื้นที่': 'NN'}],\n",
       " [{'ชาย': 'NN'}],\n",
       " [{'จำนวน': 'NN'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'ความหมาย': 'NN'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'สิ่ง': 'NN'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'ชีวิต': 'NN'}],\n",
       " [{'อื่น': 'AJ'}],\n",
       " [{'ๆ': 'NN'}],\n",
       " [{'เพราะ': 'CC'}],\n",
       " [{'อาหาร': 'NN'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'คือ': 'VV'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'กีบ': 'NN'}],\n",
       " [{'เช่น': 'CC'}],\n",
       " [{'เก้ง': 'VV'}],\n",
       " [{'กวาง': 'NN'}],\n",
       " [{'วัว': 'NN'}],\n",
       " [{'ลูก': 'NN'}],\n",
       " [{'กระทิง': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'เมื่อ': 'CC'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'อยู่': 'VV'}],\n",
       " [{'ได้': 'AX'}],\n",
       " [{'ก็': 'CC'}],\n",
       " [{'แสดง': 'VV'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'ต้อง': 'AX'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'กีบ': 'NN'}],\n",
       " [{'อาศัย': 'VV'}],\n",
       " [{'อยู่': 'VV'}],\n",
       " [{'เพียงพอ': 'VV'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'หมายถึง': 'VV'}],\n",
       " [{'พืช': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'อาหาร': 'NN'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'กีบ': 'NN'}],\n",
       " [{'เหล่า': 'CL'}],\n",
       " [{'นี้': 'AJ'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'อย่าง': 'FX'}],\n",
       " [{'อุดมสมบูรณ์': 'VV'}],\n",
       " [{'เพียงพอ': 'VV'}],\n",
       " [{'เช่น': 'CC'}],\n",
       " [{'กัน': 'PR'}],\n",
       " [{'ก็': 'CC'}],\n",
       " [{'แสดง': 'VV'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'สมบูรณ์': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'แต่': 'CC'}],\n",
       " [{'เรา': 'PR'}],\n",
       " [{'ก็': 'CC'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'เป้าหมาย': 'NN'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'เพิ่ม': 'VV'}],\n",
       " [{'ความ': 'FX'}],\n",
       " [{'อุดมสมบูรณ์': 'VV'}],\n",
       " [{'ให้': 'CC'}],\n",
       " [{'ไป': 'VV'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'เสือโคร่ง': 'NN'}],\n",
       " [{'5': 'NU'}],\n",
       " [{'ตัว': 'NN'}],\n",
       " [{'ต่อ': 'PS'}],\n",
       " [{'100': 'NU'}],\n",
       " [{'ตาราง': 'NN'}],\n",
       " [{'ภาย': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'5': 'NU'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'นาง': 'NN'}],\n",
       " [{'รตยา': 'NN'}],\n",
       " [{'กล่าว': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาง': 'NN'}],\n",
       " [{'รตยา': 'NN'}],\n",
       " [{'ซึ่ง': 'CC'}],\n",
       " [{'อยู่': 'VV'}],\n",
       " [{'ระหว่าง': 'PS'}],\n",
       " [{'ร่วม': 'VV'}],\n",
       " [{'งาน': 'NN'}],\n",
       " [{'วัน': 'NN'}],\n",
       " [{'รำลึก': 'VV'}],\n",
       " [{'17': 'NU'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'ณ': 'NN'}],\n",
       " [{'อาคาร': 'NN'}],\n",
       " [{'อนุสรณ์สถาน': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'เขต': 'NN'}],\n",
       " [{'รักษา': 'VV'}],\n",
       " [{'พันธุ์': 'NN'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ห้วยขาแข้ง': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'อำเภอ': 'NN'}],\n",
       " [{'ลานสัก': 'NN'}],\n",
       " [{'จังหวัด': 'NN'}],\n",
       " [{'อุทัยธานี': 'NN'}],\n",
       " [{'กล่าว': 'VV'}],\n",
       " [{'ด้วย': 'PS'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'แม้': 'CC'}],\n",
       " [{'แนวโน้ม': 'NN'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'อนุรักษ์': 'NN'}],\n",
       " [{'ฟื้นฟู': 'VV'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'ไป': 'VV'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'ทาง': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'ดี': 'VV'}],\n",
       " [{'ขึ้น': 'VV'}],\n",
       " [{'แต่': 'CC'}],\n",
       " [{'ก็': 'CC'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'ต้อง': 'AX'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'ทำ': 'VV'}],\n",
       " [{'ความ': 'FX'}],\n",
       " [{'เข้าใจ': 'VV'}],\n",
       " [{'กับ': 'PS'}],\n",
       " [{'หลาย': 'AJ'}],\n",
       " [{'ฝ่าย': 'NN'}],\n",
       " [{'อีก': 'AV'}],\n",
       " [{'มาก': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'โดย': 'CC'}],\n",
       " [{'เฉพาะ': 'VV'}],\n",
       " [{'ภาค': 'NN'}],\n",
       " [{'รัฐ': 'NN'}],\n",
       " [{'มัก': 'AV'}],\n",
       " [{'ดำเนิน': 'VV'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'ก้ำกึ่ง': 'NN'}],\n",
       " [{'ด้าน': 'NN'}],\n",
       " [{'หนึ่ง': 'NU'}],\n",
       " [{'อนุรักษ์': 'NN'}],\n",
       " [{'แต่': 'CC'}],\n",
       " [{'อีก': 'AV'}],\n",
       " [{'ด้าน': 'NN'}],\n",
       " [{'หนึ่ง': 'NU'}],\n",
       " [{'ก็': 'CC'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'ส่วน': 'CC'}],\n",
       " [{'ทำ': 'VV'}],\n",
       " [{'ให้': 'CC'}],\n",
       " [{'ผืน': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'หมด': 'VV'}],\n",
       " [{'ไป': 'VV'}],\n",
       " [{'เช่น': 'CC'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'อุโมงค์': 'NN'}],\n",
       " [{'ผัน': 'NN'}],\n",
       " [{'น้ำ': 'NN'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'เขื่อน': 'NN'}],\n",
       " [{'ศรีนครินทร์': 'NN'}],\n",
       " [{'ผ่าน': 'VV'}],\n",
       " [{'พื้นที่': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'มา': 'VV'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'จังหวัด': 'NN'}],\n",
       " [{'สุพรรณบุรี': 'NN'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'สร้าง': 'VV'}],\n",
       " [{'เขื่อน': 'NN'}],\n",
       " [{'แม่วงก์': 'NN'}],\n",
       " [{'จังหวัด': 'NN'}],\n",
       " [{'นครสวรรค์': 'NN'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'ส่งเสริม': 'VV'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'ท่องเที่ยว': 'VV'}],\n",
       " [{'บาง': 'NN'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'ส่งเสริม': 'VV'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'ปลูก': 'VV'}],\n",
       " [{'พืช': 'NN'}],\n",
       " [{'เชิง': 'NN'}],\n",
       " [{'เดี่ยว': 'NU'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'ตัด': 'VV'}],\n",
       " [{'ถนน': 'NN'}],\n",
       " [{'ผ่าน': 'VV'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'ลักลอบ': 'VV'}],\n",
       " [{'ตัด': 'VV'}],\n",
       " [{'ไม้': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'ปัจจุบัน': 'NN'}],\n",
       " [{'เหลือ': 'VV'}],\n",
       " [{'ไม่': 'NG'}],\n",
       " [{'มาก': 'VV'}],\n",
       " [{'แล้ว': 'AX'}],\n",
       " [{'เนื่องจาก': 'CC'}],\n",
       " [{'ไม้': 'NN'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'ใหญ่': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ถึงอย่างไร': 'CC'}],\n",
       " [{'ก็': 'CC'}],\n",
       " [{'คง': 'AX'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'ไม่': 'NG'}],\n",
       " [{'หมด': 'VV'}],\n",
       " [{'ไป': 'VV'}],\n",
       " [{'เท่า': 'VV'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'คน': 'NN'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'กิเลส': 'NN'}],\n",
       " [{'แต่': 'CC'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'ปัจจุบัน': 'NN'}],\n",
       " [{'สิ่ง': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'ส่ง': 'VV'}],\n",
       " [{'ผล': 'NN'}],\n",
       " [{'กระทบ': 'VV'}],\n",
       " [{'ต่อ': 'PS'}],\n",
       " [{'พื้นที่': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'วง': 'NN'}],\n",
       " [{'กว้าง': 'VV'}],\n",
       " [{'กว่า': 'AV'}],\n",
       " [{'คือ': 'VV'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'นโยบาย': 'NN'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'รัฐบาล': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'ขนาด': 'NN'}],\n",
       " [{'ใหญ่': 'VV'}],\n",
       " [{'เมื่อ': 'CC'}],\n",
       " [{'ดำเนิน': 'VV'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'กิน': 'VV'}],\n",
       " [{'อาณาเขต': 'NN'}],\n",
       " [{'พื้นที่': 'NN'}],\n",
       " [{'กว้างขวาง': 'VV'}],\n",
       " [{'ทั้ง': 'CC'}],\n",
       " [{'นโยบาย': 'NN'}],\n",
       " [{'ทาง': 'NN'}],\n",
       " [{'ตรง': 'VV'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'ส่งเสริม': 'VV'}],\n",
       " [{'ทาง': 'NN'}],\n",
       " [{'อ้อม': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ส่วน': 'CC'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'บุกรุก': 'VV'}],\n",
       " [{'พื้นที่': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'ชาว': 'NN'}],\n",
       " [{'บ้าน': 'NN'}],\n",
       " [{'ชุมชน': 'NN'}],\n",
       " [{'นั้น': 'AJ'}],\n",
       " [{'ส่วน': 'CC'}],\n",
       " [{'ใหญ่': 'VV'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'รู้': 'VV'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'ขอบเขต': 'NN'}],\n",
       " [{'อย่างไร': 'AV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ประธาน': 'NN'}],\n",
       " [{'มูลนิธิ': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'ฯ': 'PU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'กล่าว': 'VV'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'สำหรับ': 'PS'}],\n",
       " [{'ภาค': 'NN'}],\n",
       " [{'ประชาชน': 'NN'}],\n",
       " [{'เชื่อ': 'VV'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'ส่วน': 'CC'}],\n",
       " [{'ใหญ่': 'VV'}],\n",
       " [{'เข้าใจ': 'VV'}],\n",
       " [{'แล้ว': 'AX'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'รักษา': 'VV'}],\n",
       " [{'ผืน': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'ความ': 'FX'}],\n",
       " [{'จำเป็น': 'VV'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'กำลัง': 'AX'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'หมด': 'VV'}],\n",
       " [{'ลง': 'VV'}],\n",
       " [{'ทั้งนี้': 'CC'}],\n",
       " [{'ยัง': 'AX'}],\n",
       " [{'ต้องการ': 'VV'}],\n",
       " [{'ความ': 'FX'}],\n",
       " [{'ร่วมมือ': 'VV'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'สื่อ': 'NN'}],\n",
       " [{'มวล': 'NN'}],\n",
       " [{'ชน': 'VV'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'ทุก': 'AJ'}],\n",
       " [{'ภาค': 'NN'}],\n",
       " [{'ส่วน': 'CC'}],\n",
       " [{'ด้วย': 'PS'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ประธาน': 'NN'}],\n",
       " [{'มูลนิธิ': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'ฯ': 'PU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'กล่าว': 'VV'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'ต่อไป': 'AV'}],\n",
       " [{'ทาง': 'NN'}],\n",
       " [{'มูลนิธิ': 'NN'}],\n",
       " [{'ฯ': 'PU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'เฝ้า': 'VV'}],\n",
       " [{'ระวัง': 'VV'}],\n",
       " [{'จับตา': 'VV'}],\n",
       " [{'กฎหมาย': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'อาจ': 'NN'}],\n",
       " [{'ส่ง': 'VV'}],\n",
       " [{'ผล': 'NN'}],\n",
       " [{'กระทบ': 'VV'}],\n",
       " [{'ต่อ': 'PS'}],\n",
       " [{'ผืน': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'เปิด': 'VV'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'จำหน่าย': 'VV'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'โครงการ': 'NN'}],\n",
       " [{'ขนาด': 'NN'}],\n",
       " [{'ใหญ่': 'VV'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'ภาค': 'NN'}],\n",
       " [{'รัฐ': 'NN'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'เสริมสร้าง': 'VV'}],\n",
       " [{'ความ': 'FX'}],\n",
       " [{'เข้าใจ': 'VV'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'ชุมชน': 'NN'}],\n",
       " [{'กับ': 'PS'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ซึ่ง': 'CC'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'กลไก': 'NN'}],\n",
       " [{'สำคัญ': 'VV'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'รักษา': 'VV'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ผู้': 'FX'}],\n",
       " [{'สื่อ': 'NN'}],\n",
       " [{'ข่าว': 'NN'}],\n",
       " [{'รายงาน': 'VV'}],\n",
       " [{'ว่า': 'CC'}],\n",
       " [{'สำหรับ': 'PS'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'วัน': 'NN'}],\n",
       " [{'นี้': 'AJ'}],\n",
       " [{'(': 'PU'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ก.ย.': 'NN'}],\n",
       " [{')': 'PU'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'งาน': 'NN'}],\n",
       " [{'วัน': 'NN'}],\n",
       " [{'รำลึก': 'VV'}],\n",
       " [{'17': 'NU'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'ณ': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'อาคาร': 'NN'}],\n",
       " [{'อนุสรณ์สถาน': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'เขต': 'NN'}],\n",
       " [{'รักษา': 'VV'}],\n",
       " [{'พันธุ์': 'NN'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'ห้วยขาแข้ง': 'NN'}],\n",
       " [{'อำเภอ': 'NN'}],\n",
       " [{'ลานสัก': 'NN'}],\n",
       " [{'จังหวัด': 'NN'}],\n",
       " [{'อุทัยธานี': 'NN'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'กิจกรรม': 'NN'}],\n",
       " [{'ตั้งแต่': 'PS'}],\n",
       " [{'เวลา': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'08.00': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'น.': 'NN'}],\n",
       " [{'ประกอบ': 'VV'}],\n",
       " [{'ด้วย': 'PS'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'พิธี': 'NN'}],\n",
       " [{'สงฆ์': 'NN'}],\n",
       " [{'ทำ': 'VV'}],\n",
       " [{'บุญ': 'NN'}],\n",
       " [{'ตักบาตร': 'VV'}],\n",
       " [{'อุทิศ': 'VV'}],\n",
       " [{'ส่วน': 'CC'}],\n",
       " [{'กุศล': 'NN'}],\n",
       " [{'ให้': 'CC'}],\n",
       " [{'กับ': 'PS'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'เจ้าหน้าที่': 'NN'}],\n",
       " [{'ผู้': 'FX'}],\n",
       " [{'เสียสละ': 'VV'}],\n",
       " [{'ชีวิต': 'NN'}],\n",
       " [{'เพื่อ': 'CC'}],\n",
       " [{'ปกป้อง': 'VV'}],\n",
       " [{'ทรัพยากร': 'NN'}],\n",
       " [{'ธรรมชาติ': 'NN'}],\n",
       " [{'ป่าไม้': 'NN'}],\n",
       " [{'สัตว์': 'NN'}],\n",
       " [{'ป่า': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'นั้น': 'AJ'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'พิธี': 'NN'}],\n",
       " [{'รำลึก': 'VV'}],\n",
       " [{'17': 'NU'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'ซึ่ง': 'CC'}],\n",
       " [{'จะ': 'AX'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'อ่าน': 'VV'}],\n",
       " [{'ชีวประวัติ': 'NN'}],\n",
       " [{'สืบ': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'นาคะเสถียร': 'NN'}],\n",
       " [{'กล่าว': 'VV'}],\n",
       " [{'สดุดี': 'VV'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'ยืน': 'VV'}],\n",
       " [{'สงบ': 'VV'}],\n",
       " [{'นิ่ง': 'VV'}],\n",
       " [{'เพื่อ': 'CC'}],\n",
       " [{'ไว้': 'VV'}],\n",
       " [{'อาลัย': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ระทึก': 'VV'}],\n",
       " [{'!!': 'PU'}],\n",
       " [{'ปีก': 'NN'}],\n",
       " [{'แอร์บัส': 'NN'}],\n",
       " [{'เฉี่ยว': 'VV'}],\n",
       " [{'โรง': 'NN'}],\n",
       " [{'ซ่อม': 'VV'}],\n",
       " [{'ช่วง': 'NN'}],\n",
       " [{'ขึ้น': 'VV'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'สุวรรณภูมิ': 'NN'}],\n",
       " [{'-': 'PU'}],\n",
       " [{'เชียงใหม่': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'กันยายน': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'2550': 'NU'}],\n",
       " [{'17.31': 'NU'}],\n",
       " [{'น.': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'แอร์บัส': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'เอ': 'NN'}],\n",
       " [{'380': 'NU'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'สาธิต': 'VV'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'สุวรรณภูมิ': 'NN'}],\n",
       " [{'ไป': 'VV'}],\n",
       " [{'เชียงใหม่': 'NN'}],\n",
       " [{'แต่': 'CC'}],\n",
       " [{'ระหว่าง': 'PS'}],\n",
       " [{'นำ': 'VV'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'ขึ้น': 'VV'}],\n",
       " [{'ปีก': 'NN'}],\n",
       " [{'ด้าน': 'NN'}],\n",
       " [{'ซ้าย': 'NN'}],\n",
       " [{'เฉี่ยว': 'VV'}],\n",
       " [{'ชน': 'VV'}],\n",
       " [{'อาคาร': 'NN'}],\n",
       " [{'โรง': 'NN'}],\n",
       " [{'ซ่อม': 'VV'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'จน': 'CC'}],\n",
       " [{'ปีก': 'NN'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'ได้': 'AX'}],\n",
       " [{'รับ': 'VV'}],\n",
       " [{'ความ': 'FX'}],\n",
       " [{'เสียหาย': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'กัปตัน': 'NN'}],\n",
       " [{'ยืนยัน': 'VV'}],\n",
       " [{'เดินทาง': 'VV'}],\n",
       " [{'ต่อ': 'PS'}],\n",
       " [{'จน': 'CC'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'เผย': 'VV'}],\n",
       " [{'ไทย': 'NN'}],\n",
       " [{'สั่ง': 'VV'}],\n",
       " [{'ซื้อ': 'VV'}],\n",
       " [{'6': 'NU'}],\n",
       " [{'ลำ': 'CL'}],\n",
       " [{'กำหนด': 'VV'}],\n",
       " [{'ส่ง': 'VV'}],\n",
       " [{'มอบ': 'VV'}],\n",
       " [{'ปี': 'NN'}],\n",
       " [{'54': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'(': 'PU'}],\n",
       " [{'1': 'NU'}],\n",
       " [{'ก.ย.': 'NN'}],\n",
       " [{')': 'PU'}],\n",
       " [{'เวลา': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'9.30': 'NU'}],\n",
       " [{'น.': 'NN'}],\n",
       " [{'บริษัท': 'NN'}],\n",
       " [{'แอร์บัส': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'ได้': 'AX'}],\n",
       " [{'นำ': 'VV'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'แอร์บัส': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'เอ': 'NN'}],\n",
       " [{'380': 'NU'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'พาณิชย์': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'ใหญ่': 'VV'}],\n",
       " [{'ที่สุด': 'AV'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'โลก': 'NN'}],\n",
       " [{'ออก': 'VV'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'สาธิต': 'VV'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'ท่า': 'NN'}],\n",
       " [{'อากาศยาน': 'NN'}],\n",
       " [{'สุวรรณภูมิ': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'โดย': 'CC'}],\n",
       " [{'เที่ยว': 'VV'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'ทีจี': 'NN'}],\n",
       " [{'1380': 'NU'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'กำหนด': 'VV'}],\n",
       " [{'เดินทาง': 'VV'}],\n",
       " [{'จาก': 'PS'}],\n",
       " [{'กรุงเทพ': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'เวลา': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'09.30': 'NU'}],\n",
       " [{'น.': 'NN'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'เชียงใหม่': 'NN'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'เวลา': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'10.30': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'น.': 'NN'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'โดย': 'CC'}],\n",
       " [{'เครื่อง': 'NN'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'ลำ': 'CL'}],\n",
       " [{'ดัง': 'VV'}],\n",
       " [{'กล่าว': 'VV'}],\n",
       " [{'เดินทาง': 'VV'}],\n",
       " [{'มา': 'VV'}],\n",
       " [{'ถึง': 'PS'}],\n",
       " [{'ประเทศ': 'NN'}],\n",
       " [{'ไทย': 'NN'}],\n",
       " [{'วัน': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'31': 'NU'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'สิงหาคม': 'NN'}],\n",
       " [{'ที่': 'PS'}],\n",
       " [{'ผ่าน': 'VV'}],\n",
       " [{'มา': 'VV'}],\n",
       " [{'_': 'PU'}],\n",
       " [{'และ': 'CC'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'วัน': 'NN'}],\n",
       " [{'นี้': 'AJ'}],\n",
       " [{'เป็น': 'VV'}],\n",
       " [{'ส่วน': 'CC'}],\n",
       " [{'หนึ่ง': 'NU'}],\n",
       " [{'ของ': 'PS'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'สาธิต': 'VV'}],\n",
       " [{'ใน': 'PS'}],\n",
       " [{'เอเชีย': 'NN'}],\n",
       " [{'ทั้ง': 'CC'}],\n",
       " [{'เวียดนาม': 'NN'}],\n",
       " [{'ฮ่องกง': 'NN'}],\n",
       " [{'เที่ยว': 'VV'}],\n",
       " [{'บิน': 'VV'}],\n",
       " [{'ดัง': 'VV'}],\n",
       " [{'กล่าว': 'VV'}],\n",
       " [{'มี': 'VV'}],\n",
       " [{'การ': 'FX'}],\n",
       " [{'เชิญ': 'VV'}],\n",
       " [{'แขก': 'NN'}],\n",
       " [{'ผู้': 'FX'}],\n",
       " ...]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_test_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "25535215-12f3-446c-a789-39314269458f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NN'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_test_sub[1][0][\"เครื่อง\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "fbe54f5f-2de6-4a4e-9b32-fe7a46ebb096",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c88d30afb5a4d8a89ee90a842bdf573",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/103913 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_pred_list = []\n",
    "for i in tqdm(range(len(texts_test_raw))):\n",
    "    pred_label = pred_test_sub[i][0][texts_test_raw[i]]\n",
    "    new_pred_list.append(pred_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "b6bb642b-fbd3-498e-9db1-17a8cb30ed83",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_list = []\n",
    "for i in range(1,len(texts_test_raw)+1):\n",
    "    id_list.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c8d4653c-c918-40c4-b57d-3a04356b12ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df = pd.DataFrame()\n",
    "submission_df[\"Id\"] = id_list\n",
    "submission_df[\"Predicted\"] = new_pred_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "d53b0d58-f34a-4d35-aa7c-b393aa5194b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df.to_csv(\"submission_df_nlp.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67e766e5-2d7c-4f76-801a-8b75d0dbb2b0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}