mater-castrorum.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting search result extraction process...\n",
      "Getting data from https://epigraphy.packhum.org/search?patt=%CE%BA%CE%B1%CF%83%CF%84%CF%81%CF%89%CE%BD\n",
      "Getting data from https://epigraphy.packhum.org//text/5690?hs=225-233\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'rex' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-5bd7750434e2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    250\u001b[0m \u001b[0msearch_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSearchResultsPage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSEARCH_URL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m \u001b[0;31m# extract data and write to csv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 252\u001b[0;31m \u001b[0msearch_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_to_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Mater Castrorum.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-23-5bd7750434e2>\u001b[0m in \u001b[0;36mextract_to_csv\u001b[0;34m(self, filename)\u001b[0m\n\u001b[1;32m    231\u001b[0m         \u001b[0;34m\"\"\"Extracts all info from results and writes to a csv. Only method needed to do everything.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Starting search result extraction process...'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 233\u001b[0;31m         \u001b[0mdata_records\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    234\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Writing to {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m         \u001b[0;31m# define column header names\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-5bd7750434e2>\u001b[0m in \u001b[0;36mrecords\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    225\u001b[0m         \u001b[0mrecords\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m             \u001b[0mrecords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mrecords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-5bd7750434e2>\u001b[0m in \u001b[0;36mrecord\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    189\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;34m\"\"\"Runs extract and returns data as a dictionary.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 191\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    192\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'silent'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Extracting data from {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-5bd7750434e2>\u001b[0m in \u001b[0;36mextract\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    185\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcrossrefs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_crossrefs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrecord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-23-5bd7750434e2>\u001b[0m in \u001b[0;36mget_text\u001b[0;34m(self, as_list, simplify_text)\u001b[0m\n\u001b[1;32m    154\u001b[0m                 \u001b[0;31m# collapse redundant whitespaces\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m                 \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr'\\W+'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m                 \u001b[0mtext_line\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_line\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    157\u001b[0m                 \u001b[0;31m# remove leading & trailing whitespaces\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m                 \u001b[0mtext_line\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext_line\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'rex' is not defined"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Searches for inscriptions containing '' and outputs them as a .csv file. Intended for\n",
    "research on Julia Domna's title of ''.\n",
    "\n",
    "David J. Thomas, Instructor of History, University of South Florida\n",
    "For Julie Langford, Professor of History, University of South Florida\n",
    "\n",
    "NOT INTENDED FOR PUBLIC DISTRIBUTION\n",
    "\"\"\"\n",
    "\n",
    "\"\"\"===Imports===\"\"\"\n",
    "import re\n",
    "import csv\n",
    "import time\n",
    "from collections import UserString\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\"\"\"===Constants===\"\"\"\n",
    "\n",
    "# PHI url containing the search results\n",
    "SEARCH_URL = 'https://epigraphy.packhum.org/search?patt=%CE%BA%CE%B1%CF%83%CF%84%CF%81%CF%89%CE%BD'\n",
    "\n",
    "\"\"\"===Object Definitions===\"\"\"\n",
    "\n",
    "\n",
    "class BaseWebPage(UserString):\n",
    "    \"\"\"Parent class for all webpage scraping objects. Given a url, will provide methods for\n",
    "    requesting raw HTML from the page and converting it to a BeautifulSoup object.\"\"\"\n",
    "    html = None\n",
    "    soup = None\n",
    "    # default retries is -1 (infinite retries), set to positive integer for a retry limit\n",
    "    # silent option specifies if message should print every time a request is launched\n",
    "    # delay option specifies automated delay between web requests\n",
    "    options = { 'retries': -1, 'silent': False, 'delay': 2}\n",
    "    \n",
    "    def __init__(self, url, options={}):\n",
    "        # run parent class init function\n",
    "        super().__init__(str)\n",
    "        self.data = url\n",
    "        self.options.update(options)\n",
    "        \n",
    "    def get_response(self, retries=-1):\n",
    "        \"\"\"Fires request and returns raw response object. Retries by calling self recursively.\"\"\"\n",
    "        try:\n",
    "            print('Getting data from {}'.format(self.data))\n",
    "            # if delay is specified, pause (default is 2 seconds)\n",
    "            if self.options['delay']:\n",
    "                time.sleep(self.options['delay'])\n",
    "            return requests.get(self.data)\n",
    "        # error handling, unless retry limit is reached (if specified) call sell recursively\n",
    "        except:\n",
    "            print('Problem connecting to {}'.format(self.data))\n",
    "            # if no retry limit specified, keep calling self recursively\n",
    "            if retries == -1:\n",
    "                print('Retrying...')\n",
    "                return self.get_response()\n",
    "            # if retry limit is specified but not reached yet, call self and decrement number of attempts left\n",
    "            elif retries > 0:\n",
    "                print('Retrying... {} attempts left'.format(retries))\n",
    "                return self.get_response(retries=retries-1)\n",
    "            # otherwise, if retry limit is reached, abort\n",
    "            print('Retry limit has been reached, aborting')\n",
    "            return None\n",
    "                \n",
    "        \n",
    "    def get_html(self):\n",
    "        \"\"\"Invokes web request, extracts html string, both stores in .html and returns it.\"\"\"\n",
    "        self.html = self.get_response().text\n",
    "        return self.html\n",
    "    \n",
    "    def get_soup(self):\n",
    "        \"\"\"Parses html from url into BeautifulSoup object. Stores in .soup and returns it.\"\"\"\n",
    "        # if web request has not already been done, do it now\n",
    "        if not self.html:\n",
    "            self.get_html()\n",
    "        # if there is still not html (because of request error), abort\n",
    "        if not self.html:\n",
    "            print('Problem souping page, aborting...')\n",
    "            return None\n",
    "        # parse into BeautifulSoup object, store in self.soup and return\n",
    "        self.soup = BeautifulSoup(self.html, 'html.parser')\n",
    "        return self.soup\n",
    "\n",
    "    \n",
    "class InscriptionPage(BaseWebPage):\n",
    "    \"\"\"Extracts information from a page for a specific inscription.\"\"\"\n",
    "    reference = None\n",
    "    region = None\n",
    "    info = None\n",
    "    crossrefs = None\n",
    "    text = None\n",
    "    \n",
    "    def get_reference(self):\n",
    "        \"\"\"Gets reference number of inscription\"\"\"\n",
    "        return self.soup.find('span', class_='fullref').get_text()\n",
    "    \n",
    "    def get_region(self):\n",
    "        \"\"\"Gets general region of inscription\"\"\"\n",
    "        region_links = self.soup.find('div', class_='hdr1').find_all('a', class_='link reglink')\n",
    "        # return last item in list, which has specific region\n",
    "        return region_links[len(region_links) - 1].span.get_text()\n",
    "    \n",
    "    def get_info(self):\n",
    "        \"\"\"Gets inscription information, usually contains dating info or location information\"\"\"\n",
    "        info_text = ''\n",
    "        try:\n",
    "            info_lines = self.soup.find('div', class_='tildeinfo light').find_all('span', class_='ti')\n",
    "        # return blank string if no info found\n",
    "        except:\n",
    "            return info_text\n",
    "        for info_line in info_lines:\n",
    "            # add new information to end of info_text as well as a period and whitespace\n",
    "            info_text = info_text + info_line.get_text() + '. '\n",
    "        # remove the trailing whitespace and return\n",
    "        return info_text.rstrip()\n",
    "\n",
    "    def get_crossrefs(self, as_list=False):\n",
    "        \"\"\"Gets crossreferences, defaults to string result unless as_list specified\"\"\"\n",
    "        cross_ref_text = ''\n",
    "        try:\n",
    "            crossrefs = self.soup.find('div', class_='tildeinfo light').find_all('div', class_='xrefs')\n",
    "            # if as_list, just return crossrefs, no further work needed\n",
    "            if as_list:\n",
    "                return crossrefs\n",
    "        # return empty list or blank string if no results found\n",
    "        except:\n",
    "            if as_list:\n",
    "                return []\n",
    "            return cross_ref_text\n",
    "        for cross_ref in crossrefs:\n",
    "            # add new reference to end of cross_ref_text as well as a period and whitespace\n",
    "            cross_ref_text = cross_ref_text + cross_ref.get_text() + '. '\n",
    "        # remove the trailing whitespace and return\n",
    "        return cross_ref_text.rstrip()\n",
    "    \n",
    "    def get_text(self, as_list=False, simplify_text=True):\n",
    "        \"\"\"Gets inscription text, if as_list specified, returns as list of lines.\"\"\"\n",
    "        text = ''\n",
    "        text_lines = []\n",
    "        table_rows = self.soup.find('table', class_='grk').find_all('tr')\n",
    "        for table_row in table_rows:\n",
    "            text_lines.append(table_row.get_text())\n",
    "        # stop here if as_list specified, no further work needed\n",
    "        if as_list:\n",
    "            return text_lines\n",
    "        for text_line in text_lines:\n",
    "            # clean up text if simplify_text is flagged\n",
    "            if simplify_text == True:\n",
    "                # remove endlines\n",
    "                text_line = text_line.replace('\\n', '')\n",
    "                # remove numbers\n",
    "                text_line = ''.join([letter for letter in text_line if not letter.isdigit()])\n",
    "                # collapse redundant whitespaces\n",
    "                rex = re.compile(r'\\W+')\n",
    "                text_line = rex.sub(' ', text_line)\n",
    "                # remove leading & trailing whitespaces\n",
    "                text_line = text_line.strip()\n",
    "            # figure out if line end is hypenated\n",
    "            is_hyphenated = False\n",
    "            if text_line.endswith('-'):\n",
    "                is_hyphenated = True\n",
    "            # add new text data to end of text\n",
    "            text = text + text_line\n",
    "            # add whitespace if line is not hyphenated\n",
    "            if not is_hyphenated:\n",
    "                text = text + ' '\n",
    "            # otherwise do not add whitespace and also trim hyphen\n",
    "            else:\n",
    "                text = text[0:len(text)]\n",
    "        # remove any trailing whitespace and return\n",
    "        return text.rstrip()\n",
    "    \n",
    "    def extract(self):\n",
    "        \"\"\"Runs all getter methods and stores results in respective properties\"\"\"\n",
    "        # soup page, if not done already\n",
    "        if not self.soup:\n",
    "            self.get_soup()\n",
    "        # if there is still no soup (becauseof request error, abort)\n",
    "        if not self.soup:\n",
    "            print('Problem getting results, aborting...')\n",
    "            return None\n",
    "        self.reference = self.get_reference()\n",
    "        self.region = self.get_region()\n",
    "        self.info = self.get_info()\n",
    "        self.crossrefs = self.get_crossrefs()\n",
    "        self.text = self.get_text()\n",
    "        \n",
    "    def record(self):\n",
    "        \"\"\"Runs extract and returns data as a dictionary.\"\"\"\n",
    "        self.extract()\n",
    "        if not self.options['silent']:\n",
    "            print('Extracting data from {}'.format(self.data))\n",
    "        return {\n",
    "            'Reference': self.reference,\n",
    "            'Region': self.region,\n",
    "            'Info': self.info,\n",
    "            'Cross References': self.crossrefs,\n",
    "            'Text': self.text\n",
    "        }\n",
    "\n",
    "            \n",
    "class SearchResultsPage(BaseWebPage):\n",
    "    \"\"\"Extracts information from a search results page.\"\"\"\n",
    "    \n",
    "    def results(self):\n",
    "        \"\"\"Returns InscriptionPage objects for each matching item.\"\"\"\n",
    "        matching_items = []\n",
    "        # if web page not already souped, do so now\n",
    "        if not self.soup:\n",
    "            self.get_soup()\n",
    "        # if there is still no soup (because of request error), abort\n",
    "        if not self.soup:\n",
    "            print('Problem getting results, aborting...')\n",
    "            return None\n",
    "        matches = self.soup.find_all('div', class_='matches')\n",
    "        # loop through each result extract link, append InscriptionPage instance to list\n",
    "        for match in matches:\n",
    "            match_url = 'https://epigraphy.packhum.org/' + str(match.find('div', class_='sentr').ul.li.a['href'])\n",
    "            matching_items.append(InscriptionPage(match_url, options=self.options))\n",
    "        return matching_items\n",
    "    \n",
    "    def records(self):\n",
    "        \"\"\"Runs .results() and produces list of dictionaries with extracted data for each matching record.\"\"\"\n",
    "        records = []\n",
    "        for result in self.results():\n",
    "            records.append(result.record())\n",
    "        return records\n",
    "    \n",
    "    def extract_to_csv(self, filename='results.csv'):\n",
    "        \"\"\"Extracts all info from results and writes to a csv. Only method needed to do everything.\"\"\"\n",
    "        print('Starting search result extraction process...')\n",
    "        data_records = self.records()\n",
    "        print('Writing to {}'.format(filename))\n",
    "        # define column header names\n",
    "        col_names = ['Reference', 'Region', 'Info', 'Cross References', 'Text']\n",
    "        # open csv file, create csv writer object, and write the header row\n",
    "        with open(filename, mode='w+') as csv_file:\n",
    "            csv_writer = csv.DictWriter(csv_file, fieldnames=col_names)\n",
    "            csv_writer.writeheader()\n",
    "            # loop through each result record and write to file\n",
    "            for data_record in data_records:\n",
    "                csv_writer.writerow(data_record)\n",
    "        print('Finished writing!')\n",
    "        return True\n",
    "\n",
    "\n",
    "\"\"\"===Main Script===\"\"\"\n",
    "# create object instance\n",
    "search_results = SearchResultsPage(SEARCH_URL)\n",
    "# extract data and write to csv\n",
    "search_results.extract_to_csv(filename='Mater Castrorum.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}