UndertheseaNLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "UndertheseaNLP",
      "provenance": [],
      "authorship_tag": "ABX9TyNx0gaWfcdkWLbjTBpL99xy",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/quocthang0507/VietnameseNaturalLanguageProcessing/blob/main/UndertheseaNLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-AJm9JK1zGq-"
      },
      "source": [
        "Github repo: https://github.com/undertheseanlp/underthesea\n",
        "\n",
        "Citation:\n",
        "```\n",
        "@online{undertheseanlp/word_tokenize,\n",
        "author ={Vu Anh, Bui Nhat Anh, Doan Viet Dung},\n",
        "year = {2018},\n",
        "title ={Xây dựng hệ thống tách từ tiếng Việt},\n",
        "url ={https://github.com/undertheseanlp/word_tokenize}\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V7eCdpVtnabo",
        "outputId": "fe191149-ac5f-464d-83d9-0754337de020"
      },
      "source": [
        "!pip install underthesea"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting underthesea\n",
            "  Downloading underthesea-1.3.3-py3-none-any.whl (7.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.5 MB 5.2 MB/s \n",
            "\u001b[?25hCollecting unidecode\n",
            "  Downloading Unidecode-1.3.2-py3-none-any.whl (235 kB)\n",
            "\u001b[K     |████████████████████████████████| 235 kB 41.0 MB/s \n",
            "\u001b[?25hCollecting seqeval\n",
            "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
            "\u001b[K     |████████████████████████████████| 43 kB 1.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from underthesea) (3.13)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from underthesea) (3.2.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from underthesea) (2.23.0)\n",
            "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from underthesea) (1.9.0+cu111)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from underthesea) (1.0.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from underthesea) (0.22.2.post1)\n",
            "Requirement already satisfied: Click>=6.0 in /usr/local/lib/python3.7/dist-packages (from underthesea) (7.1.2)\n",
            "Collecting transformers>=3.5.0\n",
            "  Downloading transformers-4.11.3-py3-none-any.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 45.3 MB/s \n",
            "\u001b[?25hCollecting python-crfsuite>=0.9.6\n",
            "  Downloading python_crfsuite-0.9.7-cp37-cp37m-manylinux1_x86_64.whl (743 kB)\n",
            "\u001b[K     |████████████████████████████████| 743 kB 53.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from underthesea) (4.62.3)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.1.0->underthesea) (3.7.4.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers>=3.5.0->underthesea) (4.8.1)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 50.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers>=3.5.0->underthesea) (2019.12.20)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers>=3.5.0->underthesea) (21.0)\n",
            "Collecting PyYAML\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 47.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers>=3.5.0->underthesea) (3.3.0)\n",
            "Collecting huggingface-hub>=0.0.17\n",
            "  Downloading huggingface_hub-0.0.19-py3-none-any.whl (56 kB)\n",
            "\u001b[K     |████████████████████████████████| 56 kB 3.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers>=3.5.0->underthesea) (1.19.5)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 50.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers>=3.5.0->underthesea) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers>=3.5.0->underthesea) (3.6.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk->underthesea) (1.15.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->underthesea) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->underthesea) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->underthesea) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->underthesea) (2.10)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->underthesea) (1.4.1)\n",
            "Building wheels for collected packages: seqeval\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16181 sha256=5ebae8522a37fc83fa8e691d0e43d565c18c4debd5f86b1794f1a3a2aca5a5ac\n",
            "  Stored in directory: /root/.cache/pip/wheels/05/96/ee/7cac4e74f3b19e3158dce26a20a1c86b3533c43ec72a549fd7\n",
            "Successfully built seqeval\n",
            "Installing collected packages: PyYAML, tokenizers, sacremoses, huggingface-hub, unidecode, transformers, seqeval, python-crfsuite, underthesea\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-6.0 huggingface-hub-0.0.19 python-crfsuite-0.9.7 sacremoses-0.0.46 seqeval-1.2.2 tokenizers-0.10.3 transformers-4.11.3 underthesea-1.3.3 unidecode-1.3.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qbBLvXCWnjKx"
      },
      "source": [
        "from underthesea import sent_tokenize, word_tokenize"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g3xoIUhXnvrS"
      },
      "source": [
        "text = 'Taylor cho biết lúc đầu cô cảm thấy ngại với cô bạn thân Amanda nhưng rồi mọi thứ trôi qua nhanh chóng. Amanda cũng thoải mái với mối quan hệ này.'"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XKDR-6pXn3UY",
        "outputId": "3ea7dbcf-92d8-4e87-a3a4-ff4463c4130a"
      },
      "source": [
        "sent_tokenize(text)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Taylor cho biết lúc đầu cô cảm thấy ngại với cô bạn thân Amanda nhưng rồi mọi thứ trôi qua nhanh chóng.',\n",
              " 'Amanda cũng thoải mái với mối quan hệ này.']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PKinbffjn7dK",
        "outputId": "13c6da3b-74b0-430a-f2be-286bf1a4fffa"
      },
      "source": [
        "word_tokenize(text)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Taylor',\n",
              " 'cho',\n",
              " 'biết',\n",
              " 'lúc',\n",
              " 'đầu',\n",
              " 'cô',\n",
              " 'cảm thấy',\n",
              " 'ngại',\n",
              " 'với',\n",
              " 'cô',\n",
              " 'bạn',\n",
              " 'thân',\n",
              " 'Amanda',\n",
              " 'nhưng',\n",
              " 'rồi',\n",
              " 'mọi',\n",
              " 'thứ',\n",
              " 'trôi',\n",
              " 'qua',\n",
              " 'nhanh chóng',\n",
              " '. Amanda',\n",
              " 'cũng',\n",
              " 'thoải mái',\n",
              " 'với',\n",
              " 'mối',\n",
              " 'quan hệ',\n",
              " 'này',\n",
              " '.']"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l_2lPrr3oBPR",
        "outputId": "a12ab00f-883e-4938-d62d-23f5bf171dd5"
      },
      "source": [
        "word_tokenize('Học sinh học sinh học')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Học sinh học sinh', 'học']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    }
  ]
}