3-match_patterns_and_training_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "SQaPBd0S4Ji1"
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import pickle\n",
    "\n",
    "from spacy import displacy\n",
    "from spacy.matcher import Matcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "OxI3t0oH4WUX"
   },
   "outputs": [],
   "source": [
    "#instantiate an empty nlp with no pipeline components\n",
    "nlp = spacy.blank('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ben9ca1w4nk-",
    "outputId": "3059bef6-86de-47bd-f9fd-065937e7071a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check if it's indeed empty\n",
    "#and it is\n",
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "vfN9mhT24phC"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/all_scripts.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "956Zgygz5yW3"
   },
   "outputs": [],
   "source": [
    "#slice examples with words to learn only (label=1)\n",
    "df1 = df.loc[df['label']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "sIiVEE-u6IgH"
   },
   "outputs": [],
   "source": [
    "df1.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "c3YnsETT6Kbg",
    "outputId": "91fad21f-faf0-4f63-e5c2-661ce33dff2e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 208 entries, 0 to 207\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   label   208 non-null    int64 \n",
      " 1   script  208 non-null    object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 3.4+ KB\n"
     ]
    }
   ],
   "source": [
    "df1.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 120
    },
    "id": "Wn-gX-dL6MeU",
    "outputId": "5d172d84-3a70-4eb5-b598-b29f4313b083"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/van/opt/miniconda3/envs/spacy/lib/python3.9/site-packages/spacy/displacy/__init__.py:189: UserWarning: [W006] No entities to visualize found in Doc object. If this is surprising to you, make sure the Doc was processed using a model that supports named entity recognition, and check the `doc.ents` property manually if necessary.\n",
      "  warnings.warn(Warnings.W006)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">In the third set I think I create chance for myself because I was playing really good and felt the ball really good</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#and naturally there is no entity to render\n",
    "displacy.render(nlp(df1['script'][0]), style='ent')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LzeebRoH6iQQ"
   },
   "source": [
    "## Create Match Patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "ZArR-oEx6Pnt"
   },
   "outputs": [],
   "source": [
    "#type 1. tournament titles\n",
    "fedcup_pattern = [{'IS_TITLE': True},{'LOWER': 'cup'}]\n",
    "grand_slam_pattern = [{'LOWER': 'grand'}, {'LOWER': {'IN': ['slam', 'slams']}}]\n",
    "tournament_pattern = [{'IS_ALPHA': True, 'POS': 'PROPN'}, {'LOWER': 'open'}]\n",
    "\n",
    "#type 2. player names\n",
    "first_name_pattern = [{'POS': 'PROPN', 'IS_TITLE': True}]\n",
    "full_name_pattern = [{'IS_TITLE': True}, {'IS_TITLE': True}]\n",
    "\n",
    "#type 3. tennis terms\n",
    "double_fault_pattern = [{'LOWER': 'double'}, {'LEMMA': 'fault'}]\n",
    "rally_pattern = [{'LEMMA': 'rally'}]\n",
    "set_pattern = [{'POS': 'ADJ'}, {'LOWER': 'set'}]\n",
    "dropshot_pattern = [{'LOWER': 'dropshot'}]\n",
    "serve_pattern = [{'LEMMA': 'serve'}]\n",
    "timeout_pattern = [{'LOWER': 'medical'}, {'LOWER': 'timeout'}]\n",
    "break_pattern = [{'LEMMA': 'break'}]\n",
    "round_robin_pattern = [{'LOWER': 'round'}, {'LEMMA': 'robin'}]\n",
    "ace_pattern = [{'LEMMA': {'IN': ['ace', 'volley', 'dropshot']}}]\n",
    "winner_pattern = [{'LEMMA': 'winner'}]\n",
    "break_point_pattern = [{'LOWER': 'break'}, {'LEMMA': 'point'}]\n",
    "three_setter_pattern = [{'POS': 'NUM'}, {'IS_PUNCT': True}, {'LEMMA': 'set'}]\n",
    "\n",
    "#type 4. contextual words\n",
    "agressive_pattern = [{'LEMMA': 'aggressive'}]\n",
    "\n",
    "#trial\n",
    "stroke_pattern = [{'TEXT': {\"REGEX\": \"\\w*hand?\"}}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "0Gi7YfVo6nZA"
   },
   "outputs": [],
   "source": [
    "#instantiate a matcher\n",
    "matcher = Matcher(nlp.vocab, validate=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "kvcOOWgt6p3Z"
   },
   "outputs": [],
   "source": [
    "#add the matche patterns\n",
    "#try just one match for now\n",
    "matcher.add('网球比赛名称', [fedcup_pattern, grand_slam_pattern, tournament_pattern])\n",
    "# matcher.add('网球术语', [double_fault_pattern,\n",
    "#                         rally_pattern, set_pattern, \n",
    "#                         dropshot_pattern, \n",
    "#                         serve_pattern, \n",
    "#                         timeout_pattern, \n",
    "#                         break_pattern, \n",
    "#                         round_robin_pattern, \n",
    "#                         ace_pattern, winner_pattern, \n",
    "#                         break_point_pattern, \n",
    "#                         three_setter_pattern])\n",
    "# matcher.add('球员名字', [first_name_pattern, full_name_pattern])\n",
    "# matcher.add('打得很凶', [agressive_pattern])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "umzvijVi6ry0"
   },
   "outputs": [],
   "source": [
    "def generate_train_data(text):\n",
    "    doc = nlp(text)\n",
    "    matches = matcher(doc)\n",
    "    detection = [(doc[start:end].start_char, doc[start:end].end_char, nlp.vocab.strings[idx]) for idx, start, end in matches]\n",
    "    return (doc.text, {'entities': detection})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "AyYcYbOv6vvy"
   },
   "outputs": [],
   "source": [
    "train = [generate_train_data(t) for t in df1['script'] if len(matcher(nlp(t))) > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/train_data', 'wb') as file: \n",
    "    pickle.dump(train, file)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "match_patterns.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}