EDGAR_3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the individual 10K submissions using Beautiful Soup "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### In the last part, you got the 10 K Filing Links in a csv called 10K links.csv. In this part, you will use pandas to create a dataframe.  Then you will keep only the 10-K links from 2010 and later.  Finally you will repeat the process of obtaining 10-K forms."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First, we are giving you the libraries you will need:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import urllib\n",
    "import bs4 as bs\n",
    "from bs4 import SoupStrainer\n",
    "\n",
    "df = pd.read_csv('10K links.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Your task will be to:\n",
    "#### 1. Sort the data collected in the '10K Links.csv', to get only the 10-K links from 2010 or later\n",
    "#### 2. Use Beautiful Soup to obtain the links to the 10K Submission file \n",
    "#### 3. Store the information along with the Name of the Company and Date in a dataframe\n",
    "#### 4. Store the dataframe in a csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>company_name</th>\n",
       "      <th>filing_date</th>\n",
       "      <th>filing_type</th>\n",
       "      <th>weblink</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8563</th>\n",
       "      <td>8563</td>\n",
       "      <td>BECTON DICKINSON &amp;</td>\n",
       "      <td>2010-01-21</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/10795/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1704</th>\n",
       "      <td>1704</td>\n",
       "      <td>PIEDMONT OFFICE</td>\n",
       "      <td>2010-01-22</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/104277...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4694</th>\n",
       "      <td>4694</td>\n",
       "      <td>TIBCO SOFTWARE INC</td>\n",
       "      <td>2010-01-29</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/108528...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10652</th>\n",
       "      <td>10652</td>\n",
       "      <td>KB HOME</td>\n",
       "      <td>2010-01-29</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/795266...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10584</th>\n",
       "      <td>10584</td>\n",
       "      <td>MSCI INC</td>\n",
       "      <td>2010-01-29</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/140819...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9857</th>\n",
       "      <td>9857</td>\n",
       "      <td>SKYWORKS SOLUTIONS I</td>\n",
       "      <td>2010-02-01</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/4127/0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4176</th>\n",
       "      <td>4176</td>\n",
       "      <td>CULLEN FROST BANKERS</td>\n",
       "      <td>2010-02-02</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/39263/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>566</td>\n",
       "      <td>VASCULAR SOLUTIONS INC</td>\n",
       "      <td>2010-02-02</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/103020...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>10014</td>\n",
       "      <td>ARROW ELECTRONIC</td>\n",
       "      <td>2010-02-03</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/7536/0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5416</th>\n",
       "      <td>5416</td>\n",
       "      <td>MDC HOLDINGS INC</td>\n",
       "      <td>2010-02-05</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/773141...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1893</th>\n",
       "      <td>1893</td>\n",
       "      <td>UNION PACIFIC CORP</td>\n",
       "      <td>2010-02-05</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/100885...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10811</th>\n",
       "      <td>10811</td>\n",
       "      <td>JETBLUE AIRWAYS CORP</td>\n",
       "      <td>2010-02-05</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/115846...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7952</th>\n",
       "      <td>7952</td>\n",
       "      <td>EXELON CORP</td>\n",
       "      <td>2010-02-05</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/110935...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3709</th>\n",
       "      <td>3709</td>\n",
       "      <td>FASTENAL CO</td>\n",
       "      <td>2010-02-09</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/815556...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8562</th>\n",
       "      <td>8562</td>\n",
       "      <td>BECTON DICKINSON &amp;</td>\n",
       "      <td>2010-02-09</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/10795/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3560</th>\n",
       "      <td>3560</td>\n",
       "      <td>UNITEDHEALTH GROUP I</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/731766...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>740</th>\n",
       "      <td>740</td>\n",
       "      <td>UNITEDHEALTH GROUP INC</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/731766...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6999</th>\n",
       "      <td>6999</td>\n",
       "      <td>SILICON LABORATORIES</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/103807...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3194</th>\n",
       "      <td>3194</td>\n",
       "      <td>OWENS-ILLINOIS</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/812233...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>306</td>\n",
       "      <td>KILROY REALTY CORP</td>\n",
       "      <td>2010-02-11</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/102599...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>101</td>\n",
       "      <td>UNITED TECHNOLOGIES CORP</td>\n",
       "      <td>2010-02-11</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/101829...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7085</th>\n",
       "      <td>7085</td>\n",
       "      <td>REALTY INCOME CORP</td>\n",
       "      <td>2010-02-11</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/726728...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3165</th>\n",
       "      <td>3165</td>\n",
       "      <td>UNITED TECHNOLOGIES</td>\n",
       "      <td>2010-02-11</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/101829...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>193</td>\n",
       "      <td>ROVI CORP</td>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/142445...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8320</th>\n",
       "      <td>8320</td>\n",
       "      <td>RYDER SYSTEM INC</td>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/85961/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4979</th>\n",
       "      <td>4979</td>\n",
       "      <td>RIVERBED TECHNOLOGY</td>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/135732...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8404</th>\n",
       "      <td>8404</td>\n",
       "      <td>GOOGLE</td>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/128877...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>608</th>\n",
       "      <td>608</td>\n",
       "      <td>BIOMED REALTY TRUST</td>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/128923...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7378</th>\n",
       "      <td>7378</td>\n",
       "      <td>TW TELECOM INC</td>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/105775...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3009</th>\n",
       "      <td>3009</td>\n",
       "      <td>BIOMED REALTY TRUST INC</td>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/128923...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3867</th>\n",
       "      <td>3867</td>\n",
       "      <td>MILLER HERMAN INC</td>\n",
       "      <td>2019-07-30</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/66382/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8050</th>\n",
       "      <td>8050</td>\n",
       "      <td>WORTHINGTON INDUSTRI</td>\n",
       "      <td>2019-07-30</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/108516...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4247</th>\n",
       "      <td>4247</td>\n",
       "      <td>MICROSOFT</td>\n",
       "      <td>2019-08-01</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/789019...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8284</th>\n",
       "      <td>8284</td>\n",
       "      <td>MICROSOFT CORP</td>\n",
       "      <td>2019-08-01</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/789019...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1396</th>\n",
       "      <td>1396</td>\n",
       "      <td>BROADRIDGE FINANCIAL SOLUTION</td>\n",
       "      <td>2019-08-06</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/138331...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3307</th>\n",
       "      <td>3307</td>\n",
       "      <td>BROADRIDGE FINANCIAL</td>\n",
       "      <td>2019-08-06</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/138331...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3101</th>\n",
       "      <td>3101</td>\n",
       "      <td>PROCTER &amp; GAMBLE CO/THE</td>\n",
       "      <td>2019-08-06</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/80424/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8018</th>\n",
       "      <td>8018</td>\n",
       "      <td>ROYAL GOLD INC</td>\n",
       "      <td>2019-08-08</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/85535/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1639</th>\n",
       "      <td>1639</td>\n",
       "      <td>AUTOMATIC DATA PROCE</td>\n",
       "      <td>2019-08-09</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/8670/0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5682</th>\n",
       "      <td>5682</td>\n",
       "      <td>KENNAMETAL INC</td>\n",
       "      <td>2019-08-13</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/55242/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5247</th>\n",
       "      <td>5247</td>\n",
       "      <td>AVNET INC</td>\n",
       "      <td>2019-08-15</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/8858/0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6121</th>\n",
       "      <td>6121</td>\n",
       "      <td>COACH INC</td>\n",
       "      <td>2019-08-15</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/111613...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8128</th>\n",
       "      <td>8128</td>\n",
       "      <td>CARDINAL HEALTH INC</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/721371...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>576</th>\n",
       "      <td>576</td>\n",
       "      <td>LAM RESEARCH CORP</td>\n",
       "      <td>2019-08-20</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/707549...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8748</th>\n",
       "      <td>8748</td>\n",
       "      <td>BRINKER INTERNATIONA</td>\n",
       "      <td>2019-08-22</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/703351...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3490</th>\n",
       "      <td>3490</td>\n",
       "      <td>REGIS CORP</td>\n",
       "      <td>2019-08-27</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/716643...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10900</th>\n",
       "      <td>10900</td>\n",
       "      <td>WESTERN DIGITAL CORP</td>\n",
       "      <td>2019-08-27</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/106040...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8803</th>\n",
       "      <td>8803</td>\n",
       "      <td>CARPENTER TECHNOLOGY</td>\n",
       "      <td>2019-08-29</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/17843/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8049</th>\n",
       "      <td>8049</td>\n",
       "      <td>WORTHINGTON INDUSTRI</td>\n",
       "      <td>2019-08-30</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/108516...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7345</th>\n",
       "      <td>7345</td>\n",
       "      <td>CISCO</td>\n",
       "      <td>2019-09-05</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/858877...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6358</th>\n",
       "      <td>6358</td>\n",
       "      <td>COPART INC</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/900075...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2111</th>\n",
       "      <td>2111</td>\n",
       "      <td>THOR INDUSTRIES INC</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/730263...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8183</th>\n",
       "      <td>8183</td>\n",
       "      <td>UNITED NATURAL FOODS</td>\n",
       "      <td>2019-10-01</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/102085...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3038</th>\n",
       "      <td>3038</td>\n",
       "      <td>ASCENA RETAIL GROUP</td>\n",
       "      <td>2019-10-10</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/149830...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3958</th>\n",
       "      <td>3958</td>\n",
       "      <td>COSTCO</td>\n",
       "      <td>2019-10-11</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/909832...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6726</th>\n",
       "      <td>6726</td>\n",
       "      <td>MICRON TECHNOLOGY INC</td>\n",
       "      <td>2019-10-17</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/723125...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9561</th>\n",
       "      <td>9561</td>\n",
       "      <td>MICRON TECHNOLOGY</td>\n",
       "      <td>2019-10-17</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/723125...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6811</th>\n",
       "      <td>6811</td>\n",
       "      <td>JABIL CIRCUIT INC</td>\n",
       "      <td>2019-10-22</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/898293...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3047</th>\n",
       "      <td>3047</td>\n",
       "      <td>MSC INDUSTRIAL DIREC</td>\n",
       "      <td>2019-10-24</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/100307...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6754</th>\n",
       "      <td>6754</td>\n",
       "      <td>FACTSET RESEARCH SYS</td>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>10-K</td>\n",
       "      <td>https://www.sec.gov/Archives/edgar/data/101323...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4490 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0                   company_name filing_date filing_type  \\\n",
       "8563         8563             BECTON DICKINSON &  2010-01-21      10-K/A   \n",
       "1704         1704                PIEDMONT OFFICE  2010-01-22      10-K/A   \n",
       "4694         4694             TIBCO SOFTWARE INC  2010-01-29        10-K   \n",
       "10652       10652                        KB HOME  2010-01-29        10-K   \n",
       "10584       10584                       MSCI INC  2010-01-29        10-K   \n",
       "9857         9857           SKYWORKS SOLUTIONS I  2010-02-01      10-K/A   \n",
       "4176         4176           CULLEN FROST BANKERS  2010-02-02        10-K   \n",
       "566           566         VASCULAR SOLUTIONS INC  2010-02-02        10-K   \n",
       "10014       10014               ARROW ELECTRONIC  2010-02-03        10-K   \n",
       "5416         5416               MDC HOLDINGS INC  2010-02-05        10-K   \n",
       "1893         1893             UNION PACIFIC CORP  2010-02-05        10-K   \n",
       "10811       10811           JETBLUE AIRWAYS CORP  2010-02-05        10-K   \n",
       "7952         7952                    EXELON CORP  2010-02-05        10-K   \n",
       "3709         3709                    FASTENAL CO  2010-02-09        10-K   \n",
       "8562         8562             BECTON DICKINSON &  2010-02-09      10-K/A   \n",
       "3560         3560           UNITEDHEALTH GROUP I  2010-02-10        10-K   \n",
       "740           740         UNITEDHEALTH GROUP INC  2010-02-10        10-K   \n",
       "6999         6999           SILICON LABORATORIES  2010-02-10        10-K   \n",
       "3194         3194                 OWENS-ILLINOIS  2010-02-10        10-K   \n",
       "306           306             KILROY REALTY CORP  2010-02-11        10-K   \n",
       "101           101       UNITED TECHNOLOGIES CORP  2010-02-11        10-K   \n",
       "7085         7085             REALTY INCOME CORP  2010-02-11        10-K   \n",
       "3165         3165            UNITED TECHNOLOGIES  2010-02-11        10-K   \n",
       "193           193                      ROVI CORP  2010-02-12        10-K   \n",
       "8320         8320               RYDER SYSTEM INC  2010-02-12        10-K   \n",
       "4979         4979            RIVERBED TECHNOLOGY  2010-02-12        10-K   \n",
       "8404         8404                         GOOGLE  2010-02-12        10-K   \n",
       "608           608            BIOMED REALTY TRUST  2010-02-12        10-K   \n",
       "7378         7378                 TW TELECOM INC  2010-02-12        10-K   \n",
       "3009         3009        BIOMED REALTY TRUST INC  2010-02-12        10-K   \n",
       "...           ...                            ...         ...         ...   \n",
       "3867         3867              MILLER HERMAN INC  2019-07-30        10-K   \n",
       "8050         8050           WORTHINGTON INDUSTRI  2019-07-30        10-K   \n",
       "4247         4247                      MICROSOFT  2019-08-01        10-K   \n",
       "8284         8284                 MICROSOFT CORP  2019-08-01        10-K   \n",
       "1396         1396  BROADRIDGE FINANCIAL SOLUTION  2019-08-06        10-K   \n",
       "3307         3307           BROADRIDGE FINANCIAL  2019-08-06        10-K   \n",
       "3101         3101        PROCTER & GAMBLE CO/THE  2019-08-06        10-K   \n",
       "8018         8018                 ROYAL GOLD INC  2019-08-08        10-K   \n",
       "1639         1639           AUTOMATIC DATA PROCE  2019-08-09        10-K   \n",
       "5682         5682                 KENNAMETAL INC  2019-08-13        10-K   \n",
       "5247         5247                      AVNET INC  2019-08-15        10-K   \n",
       "6121         6121                      COACH INC  2019-08-15        10-K   \n",
       "8128         8128            CARDINAL HEALTH INC  2019-08-20        10-K   \n",
       "576           576              LAM RESEARCH CORP  2019-08-20        10-K   \n",
       "8748         8748           BRINKER INTERNATIONA  2019-08-22        10-K   \n",
       "3490         3490                     REGIS CORP  2019-08-27        10-K   \n",
       "10900       10900           WESTERN DIGITAL CORP  2019-08-27        10-K   \n",
       "8803         8803           CARPENTER TECHNOLOGY  2019-08-29        10-K   \n",
       "8049         8049           WORTHINGTON INDUSTRI  2019-08-30      10-K/A   \n",
       "7345         7345                          CISCO  2019-09-05        10-K   \n",
       "6358         6358                     COPART INC  2019-09-30        10-K   \n",
       "2111         2111            THOR INDUSTRIES INC  2019-09-30        10-K   \n",
       "8183         8183           UNITED NATURAL FOODS  2019-10-01        10-K   \n",
       "3038         3038            ASCENA RETAIL GROUP  2019-10-10        10-K   \n",
       "3958         3958                         COSTCO  2019-10-11        10-K   \n",
       "6726         6726          MICRON TECHNOLOGY INC  2019-10-17        10-K   \n",
       "9561         9561              MICRON TECHNOLOGY  2019-10-17        10-K   \n",
       "6811         6811              JABIL CIRCUIT INC  2019-10-22        10-K   \n",
       "3047         3047           MSC INDUSTRIAL DIREC  2019-10-24        10-K   \n",
       "6754         6754           FACTSET RESEARCH SYS  2019-10-30        10-K   \n",
       "\n",
       "                                                 weblink  \n",
       "8563   https://www.sec.gov/Archives/edgar/data/10795/...  \n",
       "1704   https://www.sec.gov/Archives/edgar/data/104277...  \n",
       "4694   https://www.sec.gov/Archives/edgar/data/108528...  \n",
       "10652  https://www.sec.gov/Archives/edgar/data/795266...  \n",
       "10584  https://www.sec.gov/Archives/edgar/data/140819...  \n",
       "9857   https://www.sec.gov/Archives/edgar/data/4127/0...  \n",
       "4176   https://www.sec.gov/Archives/edgar/data/39263/...  \n",
       "566    https://www.sec.gov/Archives/edgar/data/103020...  \n",
       "10014  https://www.sec.gov/Archives/edgar/data/7536/0...  \n",
       "5416   https://www.sec.gov/Archives/edgar/data/773141...  \n",
       "1893   https://www.sec.gov/Archives/edgar/data/100885...  \n",
       "10811  https://www.sec.gov/Archives/edgar/data/115846...  \n",
       "7952   https://www.sec.gov/Archives/edgar/data/110935...  \n",
       "3709   https://www.sec.gov/Archives/edgar/data/815556...  \n",
       "8562   https://www.sec.gov/Archives/edgar/data/10795/...  \n",
       "3560   https://www.sec.gov/Archives/edgar/data/731766...  \n",
       "740    https://www.sec.gov/Archives/edgar/data/731766...  \n",
       "6999   https://www.sec.gov/Archives/edgar/data/103807...  \n",
       "3194   https://www.sec.gov/Archives/edgar/data/812233...  \n",
       "306    https://www.sec.gov/Archives/edgar/data/102599...  \n",
       "101    https://www.sec.gov/Archives/edgar/data/101829...  \n",
       "7085   https://www.sec.gov/Archives/edgar/data/726728...  \n",
       "3165   https://www.sec.gov/Archives/edgar/data/101829...  \n",
       "193    https://www.sec.gov/Archives/edgar/data/142445...  \n",
       "8320   https://www.sec.gov/Archives/edgar/data/85961/...  \n",
       "4979   https://www.sec.gov/Archives/edgar/data/135732...  \n",
       "8404   https://www.sec.gov/Archives/edgar/data/128877...  \n",
       "608    https://www.sec.gov/Archives/edgar/data/128923...  \n",
       "7378   https://www.sec.gov/Archives/edgar/data/105775...  \n",
       "3009   https://www.sec.gov/Archives/edgar/data/128923...  \n",
       "...                                                  ...  \n",
       "3867   https://www.sec.gov/Archives/edgar/data/66382/...  \n",
       "8050   https://www.sec.gov/Archives/edgar/data/108516...  \n",
       "4247   https://www.sec.gov/Archives/edgar/data/789019...  \n",
       "8284   https://www.sec.gov/Archives/edgar/data/789019...  \n",
       "1396   https://www.sec.gov/Archives/edgar/data/138331...  \n",
       "3307   https://www.sec.gov/Archives/edgar/data/138331...  \n",
       "3101   https://www.sec.gov/Archives/edgar/data/80424/...  \n",
       "8018   https://www.sec.gov/Archives/edgar/data/85535/...  \n",
       "1639   https://www.sec.gov/Archives/edgar/data/8670/0...  \n",
       "5682   https://www.sec.gov/Archives/edgar/data/55242/...  \n",
       "5247   https://www.sec.gov/Archives/edgar/data/8858/0...  \n",
       "6121   https://www.sec.gov/Archives/edgar/data/111613...  \n",
       "8128   https://www.sec.gov/Archives/edgar/data/721371...  \n",
       "576    https://www.sec.gov/Archives/edgar/data/707549...  \n",
       "8748   https://www.sec.gov/Archives/edgar/data/703351...  \n",
       "3490   https://www.sec.gov/Archives/edgar/data/716643...  \n",
       "10900  https://www.sec.gov/Archives/edgar/data/106040...  \n",
       "8803   https://www.sec.gov/Archives/edgar/data/17843/...  \n",
       "8049   https://www.sec.gov/Archives/edgar/data/108516...  \n",
       "7345   https://www.sec.gov/Archives/edgar/data/858877...  \n",
       "6358   https://www.sec.gov/Archives/edgar/data/900075...  \n",
       "2111   https://www.sec.gov/Archives/edgar/data/730263...  \n",
       "8183   https://www.sec.gov/Archives/edgar/data/102085...  \n",
       "3038   https://www.sec.gov/Archives/edgar/data/149830...  \n",
       "3958   https://www.sec.gov/Archives/edgar/data/909832...  \n",
       "6726   https://www.sec.gov/Archives/edgar/data/723125...  \n",
       "9561   https://www.sec.gov/Archives/edgar/data/723125...  \n",
       "6811   https://www.sec.gov/Archives/edgar/data/898293...  \n",
       "3047   https://www.sec.gov/Archives/edgar/data/100307...  \n",
       "6754   https://www.sec.gov/Archives/edgar/data/101323...  \n",
       "\n",
       "[4490 rows x 5 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values('filing_date', inplace = True)\n",
    "df = df.loc[df['filing_date'] > '2010-01-01']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\iamak\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:32: DeprecationWarning: MyOpener style of invoking requests is deprecated. Use newer urlopen functions/methods\n"
     ]
    }
   ],
   "source": [
    "weblink = []\n",
    "filing_type = []\n",
    "company_name = []\n",
    "date = []\n",
    "\n",
    "df = pd.read_csv('Dropped_Companies.csv')\n",
    "\n",
    "company_name = df['Name of Stock'].tolist()\n",
    "for i in range(0, len(company_name)):\n",
    "    company_name[i] = company_name[i].replace(\" \", \"+\")\n",
    "\n",
    "import time\n",
    "from urllib.request import FancyURLopener  # This is library that helps us create the headless browser\n",
    "from random import choice #This library helps pick a random item from a list\n",
    "\n",
    "user_agents = [\n",
    "    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',\n",
    "    'Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16',\n",
    "    'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',\n",
    "    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A',\n",
    "    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246'\n",
    "]\n",
    "\n",
    "# These are the usr agents for each of different browsers. Here we are using five, but it can be any number of user agents\n",
    "class MyOpener(FancyURLopener, object):\n",
    "    version = choice(user_agents)\n",
    "\n",
    "\n",
    "    \n",
    "link = 'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&company=FMC+CORP&type=10-K&dateb=&owner=exclude&count=100'\n",
    "\n",
    "myopener = MyOpener()\n",
    "page=myopener.open(link)\n",
    "\n",
    "html = page.read().decode('utf-8')\n",
    "description = []\n",
    "soup = bs.BeautifulSoup(html)\n",
    "infotable1 = soup.find_all(\"input\", value = \"Next 100\")\n",
    "for i in infotable1:\n",
    "    description.append(i.get('onclick')[17:-1])\n",
    "    \n",
    "# soup = bs.BeautifulSoup(html)\n",
    "# infotable = soup.find_all(\"a\", id = \"documentsbutton\")\n",
    "# for link in infotable:\n",
    "#     weblink.append('https://www.sec.gov' + link.get('href'))\n",
    "\n",
    "# infotable = soup.find_all(\"table\", class_= \"tableFile2\")\n",
    "# for link in infotable:\n",
    "#     row = link.find_all(\"td\", nowrap  = \"nowrap\")\n",
    "#     for i in range(0, len(row), 3):\n",
    "#         filing_type.append(row[i].getText())\n",
    "#         date.append(link.find(\"td\", class_ = \"small\").find_next_sibling(\"td\").text)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "infotable1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for line in sys.stdin:\n",
    "    data = line.strip().split(\" \")\n",
    "    if(len(data) > 1):\n",
    "        ip = data[0]\n",
    "        print \"{0}\\t{1}\".format(ip, 1)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}