text_tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data.texts import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = book1 + book2 + book3 + book4 + book5 + book6 + book7 + book8 + book9 + book10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cltk.tokenize.latin.sentence import SentenceTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = SentenceTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "str_text = ' '.join(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_tokenized_text = tokenizer.tokenize(str_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cltk.corpus.utils.formatter import phi5_plaintext_cleanup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text filtering\n",
    "We use the plaintext cleanup to remove the punctuation and periods of the text. This makes certain tasks - such as text lemmatization - more efficient. If the text has been scraped from a webpage you might want to call ``remove_non_ascii()`` or ``remove_non_latin()`` characters to remove any remaining characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_str_text = phi5_plaintext_cleanup(str_text, rm_punctuation=True, rm_periods=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_line_tokenized_text = list(map(lambda x: phi5_plaintext_cleanup(x, rm_punctuation=True, rm_periods=True), line_tokenized_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Conditi paradoxi compositio mellis pondo XV in aeneum vas mittuntur praemissis vini sextariis duobus ut in coctura mellis vinum decoquas',\n",
       " 'quod igni lento et aridis lignis calefactum commotum ferula dum coquitur si effervere coeperit vini rore compescitur praeter quod subtracto igni in se redit',\n",
       " 'cum perfrixerit rursus accenditur',\n",
       " 'hoc secundo ac tertio fiet ac tum demum remotum a foco post pridie despumatur',\n",
       " 'tum mittes piperis uncias IV iam triti masticis scripulos III folii et croci dragmae singulae dactilorum ossibus torridis quinque isdemque dactilis vino mollitis intercedente prius suffusione vini de suo modo ac numero ut tritura lenis habeatur',\n",
       " 'his omnibus paratis supermittis vini lenis sextaria XVIII',\n",
       " 'carbones perfecto aderunt duo milia',\n",
       " 'Conditum melizomum perpetuum quod subministratur per viam peregrinanti piper tritum cum melle despumato in cupellam mittis conditi loco et ad momentum quantum sit bibendum tantum aut mellis proferas aut vinum misceas sed si vas erit nonnihil vini melizomo mittas adiciendum propter mellis exitum solutiorem',\n",
       " 'Absinthium Romanum sic facies conditi Camerini praeceptis utique pro absinthio cessante in cuius vicem absinthi Pontici purgati terendique unciam Thebaicam dabis masticis folii III costi scripulos senos croci scripulos III vini eiusmodi sextarios XVIII',\n",
       " 'carbones amaritudo non exigit']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_line_tokenized_text[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cltk.tokenize.word import WordTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_tokenizer = WordTokenizer('latin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_tokenized_text = word_tokenizer.tokenize(filtered_str_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Conditi',\n",
       " 'paradoxi',\n",
       " 'compositio',\n",
       " 'mellis',\n",
       " 'pondo',\n",
       " 'XV',\n",
       " 'in',\n",
       " 'aeneum',\n",
       " 'vas',\n",
       " 'mittuntur']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenized_text[:10]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}