006_PyLadiesMadrid_NLP_WebScrapping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrapping sobre los Goya"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## web scrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib.request\n",
    "import pandas as pd\n",
    "import time\n",
    "from random import randint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_films = []\n",
    "for myedicion in reversed(range(1,33)):\n",
    "    print(myedicion)\n",
    "    time.sleep(1)\n",
    "    myurl = 'https://www.premiosgoya.com/' + str(myedicion) + '-edicion/nominaciones/por-categoria/pelicula/'\n",
    "    with urllib.request.urlopen(myurl) as url:\n",
    "        s = url.read()\n",
    "    soup = BeautifulSoup(s, \"lxml\")\n",
    "    print(type(soup))\n",
    "    snippet = soup.find_all(\"h2\", {\"class\": \"lista-de-peliculas__titulo\"})\n",
    "    for h2 in snippet:\n",
    "        peli = str(h2.a['href'])\n",
    "        url_peli = \"https://www.premiosgoya.com\" + peli\n",
    "        mydict = {'edicion': myedicion, 'url_peli': url_peli}\n",
    "        all_films.append(mydict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(len(all_films))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seen = set()\n",
    "all_films_unique = []\n",
    "for d in all_films:\n",
    "    t = tuple(d.items())\n",
    "    if t not in seen:\n",
    "        seen.add(t)\n",
    "        all_films_unique.append(d)\n",
    "\n",
    "print(len(all_films))\n",
    "print(len(all_films_unique))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "info_pelis = []\n",
    "for peli in all_films_unique:\n",
    "    time.sleep(randint(0, 5))\n",
    "    with urllib.request.urlopen(peli['url_peli']) as url:\n",
    "        s = url.read()\n",
    "    soup = BeautifulSoup(s, \"lxml\")\n",
    "    mytittles = soup.find_all('h1', {\"class\":\"pelicula__header__titulo\"})\n",
    "    for t in mytittles:\n",
    "        peli['titulo'] = t.text\n",
    "    snipped = soup.find_all('div', {\"class\":\"pelicula__header__descripcion\"})\n",
    "    for des in snipped:\n",
    "        peli['descripcion'] =  des.text\n",
    "    info_pelis.append(peli)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "info_pelis_pd = pd.DataFrame(info_pelis)\n",
    "info_pelis_pd.to_csv(\"info_pelis_pd.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}