Spacy_Sentence_Segmentation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Spacy - Sentence Segmentation",
      "provenance": [],
      "authorship_tag": "ABX9TyNGLQJTreSLMzqem+xZ/Zyx",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/quocthang0507/VietnameseNaturalLanguageProcessing/blob/main/Spacy_Sentence_Segmentation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "1Lcl8ikJrtcq",
        "outputId": "4b04cc95-a1ad-4c3d-e601-cb21b3952404"
      },
      "source": [
        "!pip install spacy==3.0"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting spacy==3.0\n",
            "  Downloading spacy-3.0.0-cp37-cp37m-manylinux2014_x86_64.whl (12.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 12.7 MB 218 kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (1.19.5)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (3.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (1.0.5)\n",
            "Collecting srsly<3.0.0,>=2.4.0\n",
            "  Downloading srsly-2.4.1-cp37-cp37m-manylinux2014_x86_64.whl (456 kB)\n",
            "\u001b[K     |████████████████████████████████| 456 kB 48.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (4.8.1)\n",
            "Collecting pathy\n",
            "  Downloading pathy-0.6.0-py3-none-any.whl (42 kB)\n",
            "\u001b[K     |████████████████████████████████| 42 kB 1.3 MB/s \n",
            "\u001b[?25hCollecting catalogue<2.1.0,>=2.0.1\n",
            "  Downloading catalogue-2.0.6-py3-none-any.whl (17 kB)\n",
            "Collecting spacy-legacy<3.1.0,>=3.0.0\n",
            "  Downloading spacy_legacy-3.0.8-py2.py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (2.0.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (57.4.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (21.0)\n",
            "Collecting pydantic<1.8.0,>=1.7.1\n",
            "  Downloading pydantic-1.7.4-cp37-cp37m-manylinux2014_x86_64.whl (9.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 9.1 MB 14.4 MB/s \n",
            "\u001b[?25hCollecting typer<0.4.0,>=0.3.0\n",
            "  Downloading typer-0.3.2-py3-none-any.whl (21 kB)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (4.62.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (2.23.0)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (0.4.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (2.11.3)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (0.8.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy==3.0) (3.7.4.3)\n",
            "Collecting thinc<8.1.0,>=8.0.0\n",
            "  Downloading thinc-8.0.10-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (623 kB)\n",
            "\u001b[K     |████████████████████████████████| 623 kB 46.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.1->spacy==3.0) (3.5.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy==3.0) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.0) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.0) (2.10)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.4.0,>=0.3.0->spacy==3.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy==3.0) (2.0.1)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy->spacy==3.0) (5.2.1)\n",
            "Installing collected packages: catalogue, typer, srsly, pydantic, thinc, spacy-legacy, pathy, spacy\n",
            "  Attempting uninstall: catalogue\n",
            "    Found existing installation: catalogue 1.0.0\n",
            "    Uninstalling catalogue-1.0.0:\n",
            "      Successfully uninstalled catalogue-1.0.0\n",
            "  Attempting uninstall: srsly\n",
            "    Found existing installation: srsly 1.0.5\n",
            "    Uninstalling srsly-1.0.5:\n",
            "      Successfully uninstalled srsly-1.0.5\n",
            "  Attempting uninstall: thinc\n",
            "    Found existing installation: thinc 7.4.0\n",
            "    Uninstalling thinc-7.4.0:\n",
            "      Successfully uninstalled thinc-7.4.0\n",
            "  Attempting uninstall: spacy\n",
            "    Found existing installation: spacy 2.2.4\n",
            "    Uninstalling spacy-2.2.4:\n",
            "      Successfully uninstalled spacy-2.2.4\n",
            "Successfully installed catalogue-2.0.6 pathy-0.6.0 pydantic-1.7.4 spacy-3.0.0 spacy-legacy-3.0.8 srsly-2.4.1 thinc-8.0.10 typer-0.3.2\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "catalogue",
                  "spacy",
                  "srsly",
                  "thinc"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4vUy3m0upNEM"
      },
      "source": [
        "import spacy"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fTGhYg1epgV_",
        "outputId": "89f09e2b-f1ed-42d0-fc61-09579533f675"
      },
      "source": [
        "!python -m spacy download en_core_web_sm\n",
        "!python -m spacy download en_core_web_md\n",
        "!python -m spacy download en_core_web_lg"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting en-core-web-sm==3.0.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0-py3-none-any.whl (13.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 13.7 MB 72 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.1.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-sm==3.0.0) (3.0.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (57.4.0)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.0.6)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (0.8.2)\n",
            "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (4.8.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (3.0.8)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.4.1)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (0.4.1)\n",
            "Requirement already satisfied: pydantic<1.8.0,>=1.7.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (1.7.4)\n",
            "Requirement already satisfied: typer<0.4.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (0.3.2)\n",
            "Requirement already satisfied: pathy in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (0.6.0)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (4.62.2)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (8.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.0.5)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.11.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (3.0.5)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (1.19.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (21.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (3.7.4.3)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (1.0.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.1->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (3.5.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (3.0.4)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.4.0,>=0.3.0->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (2.0.1)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy->spacy<3.1.0,>=3.0.0->en-core-web-sm==3.0.0) (5.2.1)\n",
            "Installing collected packages: en-core-web-sm\n",
            "  Attempting uninstall: en-core-web-sm\n",
            "    Found existing installation: en-core-web-sm 2.2.5\n",
            "    Uninstalling en-core-web-sm-2.2.5:\n",
            "      Successfully uninstalled en-core-web-sm-2.2.5\n",
            "Successfully installed en-core-web-sm-3.0.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n",
            "Collecting en-core-web-md==3.0.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.0.0/en_core_web_md-3.0.0-py3-none-any.whl (47.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 47.1 MB 3.7 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.1.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-md==3.0.0) (3.0.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (3.0.8)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (1.0.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (57.4.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.11.3)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (0.4.1)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (8.0.10)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (3.0.5)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.23.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.0.5)\n",
            "Requirement already satisfied: pydantic<1.8.0,>=1.7.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (1.7.4)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.4.1)\n",
            "Requirement already satisfied: typer<0.4.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (0.3.2)\n",
            "Requirement already satisfied: pathy in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (0.6.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (4.62.2)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (0.8.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (3.7.4.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (21.0)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.0.6)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (4.8.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.1->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (3.5.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (3.0.4)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.4.0,>=0.3.0->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (2.0.1)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy->spacy<3.1.0,>=3.0.0->en-core-web-md==3.0.0) (5.2.1)\n",
            "Installing collected packages: en-core-web-md\n",
            "Successfully installed en-core-web-md-3.0.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_md')\n",
            "Collecting en-core-web-lg==3.0.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.0.0/en_core_web_lg-3.0.0-py3-none-any.whl (778.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 778.8 MB 10 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.1.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-lg==3.0.0) (3.0.0)\n",
            "Requirement already satisfied: pathy in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (0.6.0)\n",
            "Requirement already satisfied: typer<0.4.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (0.3.2)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (4.62.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.4.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (1.0.5)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.0.5)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (0.8.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.23.0)\n",
            "Requirement already satisfied: pydantic<1.8.0,>=1.7.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (1.7.4)\n",
            "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (4.8.1)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (3.0.5)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (3.7.4.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (57.4.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.11.3)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.0.6)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (21.0)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (0.4.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (3.0.8)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (8.0.10)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (1.19.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.1->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (3.5.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.10)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.4.0,>=0.3.0->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (2.0.1)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy->spacy<3.1.0,>=3.0.0->en-core-web-lg==3.0.0) (5.2.1)\n",
            "Installing collected packages: en-core-web-lg\n",
            "Successfully installed en-core-web-lg-3.0.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_lg')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hLO6qOHcpvug"
      },
      "source": [
        "# Default: Using the dependency parse\n",
        "\n",
        "Unlike other libraries, spaCy uses the dependency parse to determine sentence boundaries. This is usually the most accurate approach, but it requires a **trained pipeline** that provides accurate predictions. If your texts are closer to general-purpose news or web text, this should work well out-of-the-box with spaCy’s provided trained pipelines. For social media or conversational text that doesn’t follow the same rules, your application may benefit from a custom trained or rule-based component."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MiXaX1X_pWDI",
        "outputId": "5e7d3047-7d1f-4675-efca-9bae58318fb0"
      },
      "source": [
        "nlp_default = spacy.load('en_core_web_sm')\n",
        "doc = nlp_default('This is a sentence. This is another sentence.')\n",
        "for sent in doc.sents:\n",
        "    print(sent.text)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This is a sentence.\n",
            "This is another sentence.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4CShj1BAp77M"
      },
      "source": [
        "# Statistical sentence segmenter\n",
        "\n",
        "The `SentenceRecognizer` is a simple statistical component that only provides sentence boundaries. Along with being faster and smaller than the parser, its primary advantage is that it’s easier to train because it only requires annotated sentence boundaries rather than full dependency parses. spaCy’s [trained pipelines](https://spacy.io/models) include both a parser and a trained sentence segmenter, which is [disabled](https://spacy.io/usage/processing-pipelines#disabling) by default. If you only need sentence boundaries and no parser, you can use the `exclude` or `disable` argument on `spacy.load` to load the pipeline without the parser and then enable the sentence recognizer explicitly with `nlp.enable_pipe`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jdZxAG1qpuI8",
        "outputId": "4ca5f1d5-8372-4e45-f8b0-931a9600acc1"
      },
      "source": [
        "nlp_stat = spacy.load('en_core_web_sm', exclude=['parser'])\n",
        "nlp_stat.enable_pipe(\"senter\")\n",
        "doc = nlp_stat('This is a sentence. This is another sentence.')\n",
        "for sent in doc.sents:\n",
        "    print(sent.text)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This is a sentence.\n",
            "This is another sentence.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sakvm3jKscGk"
      },
      "source": [
        "# Rule-based pipeline component\n",
        "\n",
        "The `Sentencizer` component is a [pipeline component](https://spacy.io/usage/processing-pipelines) that splits sentences on punctuation like `.`, `!` or `?`. You can plug it into your pipeline if you only need sentence boundaries without dependency parses."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "50yhazZRspSy",
        "outputId": "460c27f9-45fa-435b-e002-170d49b0ecfe"
      },
      "source": [
        "from spacy.lang.en import English\n",
        "\n",
        "nlp_rule = English()\n",
        "nlp_rule.add_pipe('sentencizer')\n",
        "doc = nlp_rule('This is a sentence. This is another sentence.')\n",
        "for sent in doc.sents:\n",
        "    print(sent.text)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This is a sentence.\n",
            "This is another sentence.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n5thlglltHKs"
      },
      "source": [
        "# Custom rule-based strategy\n",
        "\n",
        "If you want to implement your own strategy that differs from the default rule-based approach of splitting on sentences, you can also create a [custom pipeline component](https://spacy.io/usage/processing-pipelines#custom-components) that takes a `Doc` object and sets the `Token.is_sent_start` attribute on each individual token. If set to `False`, the token is explicitly marked as not the start of a sentence. If set to `None` (default), it’s treated as a missing value and can still be overwritten by the parser.\n",
        "\n",
        "> **Important note**\n",
        ">\n",
        "> To prevent inconsistent state, you can only set boundaries **before** a document is parsed (and `doc.has_annotation(\"DEP\")` is `False`). To ensure that your component is added in the right place, you can set `before='parser'` or `first=True` when adding it to the pipeline using `nlp.add_pipe`.\n",
        "\n",
        "Here’s an example of a component that implements a pre-processing rule for splitting on `\"...\"` tokens. The component is added before the parser, which is then used to further segment the text. That’s possible, because `is_sent_start` is only set to `True` for some of the tokens – all others still specify `None` for unset sentence boundaries. This approach can be useful if you want to implement **additional** rules specific to your data, while still being able to take advantage of dependency-based sentence segmentation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lpqtz6nMs_J2",
        "outputId": "b0d2b135-4555-4014-e4fc-6f724ad15aee"
      },
      "source": [
        "from spacy.language import Language\n",
        "\n",
        "text = 'This is a sentence...hello...and another sentence.'\n",
        "\n",
        "nlp_custom = spacy.load('en_core_web_sm')\n",
        "doc = nlp_custom(text)\n",
        "print('Before:', [sent.text for sent in doc.sents])\n",
        "\n",
        "@Language.component('set_custom_boundaries')\n",
        "def set_custom_boundaries(doc):\n",
        "    for token in doc[:-1]:\n",
        "        if token.text == '...':\n",
        "            doc[token.i + 1].is_sent_start = True\n",
        "    return doc\n",
        "\n",
        "nlp_custom.add_pipe('set_custom_boundaries', before='parser')\n",
        "doc = nlp_custom(text)\n",
        "print('After:', [sent.text for sent in doc.sents])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Before: ['This is a sentence...hello...and another sentence.']\n",
            "After: ['This is a sentence...', 'hello...', 'and another sentence.']\n"
          ]
        }
      ]
    }
  ]
}