06 Data Filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python training for data engineers\n",
    "## 06 Data Filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goal\n",
    "Apply basic filter techniques on Pandas dataframe\n",
    "\n",
    "### Remember\n",
    "* `df.groupby(['category'])['col_name'].count()`\n",
    "* `df.sort_values(sort_col, ascending=True)`\n",
    "* `df['col_name'].nlargest(10)`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_type</th>\n",
       "      <th>filename_size_hash</th>\n",
       "      <th>package_name</th>\n",
       "      <th>python_version</th>\n",
       "      <th>uploaded_on</th>\n",
       "      <th>filename</th>\n",
       "      <th>size</th>\n",
       "      <th>unit</th>\n",
       "      <th>size_in_bytes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wheel</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27m-macosx_10_6_int...</td>\n",
       "      <td>scikit-learn</td>\n",
       "      <td>cp27</td>\n",
       "      <td>2017-10-23</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27m-macosx_10_6_int...</td>\n",
       "      <td>0</td>\n",
       "      <td>MB</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>wheel</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27m-manylinux1_i686...</td>\n",
       "      <td>scikit-learn</td>\n",
       "      <td>cp27</td>\n",
       "      <td>2017-10-23</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27m-manylinux1_i686...</td>\n",
       "      <td>4</td>\n",
       "      <td>MB</td>\n",
       "      <td>4194304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>wheel</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27m-manylinux1_x86_...</td>\n",
       "      <td>scikit-learn</td>\n",
       "      <td>cp27</td>\n",
       "      <td>2017-10-23</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27m-manylinux1_x86_...</td>\n",
       "      <td>2</td>\n",
       "      <td>MB</td>\n",
       "      <td>2097152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>wheel</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27mu-manylinux1_i68...</td>\n",
       "      <td>scikit-learn</td>\n",
       "      <td>cp27</td>\n",
       "      <td>2017-10-23</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27mu-manylinux1_i68...</td>\n",
       "      <td>4</td>\n",
       "      <td>MB</td>\n",
       "      <td>4194304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wheel</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27mu-manylinux1_x86...</td>\n",
       "      <td>scikit-learn</td>\n",
       "      <td>cp27</td>\n",
       "      <td>2017-10-23</td>\n",
       "      <td>scikit_learn-0.19.1-cp27-cp27mu-manylinux1_x86...</td>\n",
       "      <td>2</td>\n",
       "      <td>MB</td>\n",
       "      <td>2097152</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  file_type                                 filename_size_hash  package_name  \\\n",
       "0     wheel  scikit_learn-0.19.1-cp27-cp27m-macosx_10_6_int...  scikit-learn   \n",
       "1     wheel  scikit_learn-0.19.1-cp27-cp27m-manylinux1_i686...  scikit-learn   \n",
       "2     wheel  scikit_learn-0.19.1-cp27-cp27m-manylinux1_x86_...  scikit-learn   \n",
       "3     wheel  scikit_learn-0.19.1-cp27-cp27mu-manylinux1_i68...  scikit-learn   \n",
       "4     wheel  scikit_learn-0.19.1-cp27-cp27mu-manylinux1_x86...  scikit-learn   \n",
       "\n",
       "  python_version uploaded_on  \\\n",
       "0           cp27  2017-10-23   \n",
       "1           cp27  2017-10-23   \n",
       "2           cp27  2017-10-23   \n",
       "3           cp27  2017-10-23   \n",
       "4           cp27  2017-10-23   \n",
       "\n",
       "                                            filename  size unit  size_in_bytes  \n",
       "0  scikit_learn-0.19.1-cp27-cp27m-macosx_10_6_int...     0   MB              0  \n",
       "1  scikit_learn-0.19.1-cp27-cp27m-manylinux1_i686...     4   MB        4194304  \n",
       "2  scikit_learn-0.19.1-cp27-cp27m-manylinux1_x86_...     2   MB        2097152  \n",
       "3  scikit_learn-0.19.1-cp27-cp27mu-manylinux1_i68...     4   MB        4194304  \n",
       "4  scikit_learn-0.19.1-cp27-cp27mu-manylinux1_x86...     2   MB        2097152  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the XML data from the previous notebook\n",
    "import pandas as pd\n",
    "xmldf = pd.read_pickle('xml_dataframe_notebook_05.pickle')\n",
    "xmldf.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get all unique release types:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[wheel, source, windows installer]\n",
       "Categories (3, object): [wheel, source, windows installer]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xmldf['file_type'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the amount of packages per Python version:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "python_version\n",
       "2.7         3\n",
       "3.5         2\n",
       "None        1\n",
       "cp27       21\n",
       "cp33        5\n",
       "cp34       10\n",
       "cp35       15\n",
       "cp36       14\n",
       "py2.py3     4\n",
       "py3         1\n",
       "Name: package_name, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xmldf.groupby(['python_version'])['package_name'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "or just get the top 5:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "python_version\n",
       "cp27    21\n",
       "cp35    15\n",
       "cp36    14\n",
       "cp34    10\n",
       "cp33     5\n",
       "Name: package_name, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xmldf.groupby(['python_version'])['package_name'].count().nlargest(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter the files by `release_type` and sort them by size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_type</th>\n",
       "      <th>filename_size_hash</th>\n",
       "      <th>package_name</th>\n",
       "      <th>python_version</th>\n",
       "      <th>uploaded_on</th>\n",
       "      <th>filename</th>\n",
       "      <th>size</th>\n",
       "      <th>unit</th>\n",
       "      <th>size_in_bytes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>source</td>\n",
       "      <td>scikit-learn-0.19.1.tar.gz                    ...</td>\n",
       "      <td>scikit-learn</td>\n",
       "      <td>None</td>\n",
       "      <td>2017-10-23</td>\n",
       "      <td>scikit-learn-0.19.1.tar.gz                    ...</td>\n",
       "      <td>5</td>\n",
       "      <td>MB</td>\n",
       "      <td>5242880</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_type                                 filename_size_hash  package_name  \\\n",
       "22    source  scikit-learn-0.19.1.tar.gz                    ...  scikit-learn   \n",
       "\n",
       "   python_version uploaded_on  \\\n",
       "22           None  2017-10-23   \n",
       "\n",
       "                                             filename  size unit  \\\n",
       "22  scikit-learn-0.19.1.tar.gz                    ...     5   MB   \n",
       "\n",
       "    size_in_bytes  \n",
       "22        5242880  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xmldf[xmldf['file_type'] == 'source'].sort_values('size_in_bytes', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use two columns to group the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "python_version  file_type        \n",
       "2.7             wheel                 1\n",
       "                windows installer     2\n",
       "3.5             wheel                 1\n",
       "                windows installer     1\n",
       "None            source                1\n",
       "cp27            wheel                21\n",
       "cp33            wheel                 5\n",
       "cp34            wheel                10\n",
       "cp35            wheel                15\n",
       "cp36            wheel                14\n",
       "py2.py3         wheel                 4\n",
       "py3             wheel                 1\n",
       "Name: package_name, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xmldf.groupby(['python_version', 'file_type'])['package_name'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a new dataframe containing the amount of files for each package."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>package_name</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ninja</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-build</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-chem</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-ci</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-ci-addons</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-cycling</th>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-learn</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-optimize</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-ribo</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  filename\n",
       "package_name              \n",
       "ninja                   26\n",
       "scikit-build             1\n",
       "scikit-chem              2\n",
       "scikit-ci                1\n",
       "scikit-ci-addons         1\n",
       "scikit-cycling          17\n",
       "scikit-learn            26\n",
       "scikit-optimize          1\n",
       "scikit-ribo              1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_by_package = xmldf.groupby('package_name')['filename'].count().to_frame()\n",
    "count_by_package"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Which packages has the most files?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>package_name</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ninja</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-learn</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-cycling</th>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-chem</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-build</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-ci</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-ci-addons</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-optimize</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-ribo</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  filename\n",
       "package_name              \n",
       "ninja                   26\n",
       "scikit-learn            26\n",
       "scikit-cycling          17\n",
       "scikit-chem              2\n",
       "scikit-build             1\n",
       "scikit-ci                1\n",
       "scikit-ci-addons         1\n",
       "scikit-optimize          1\n",
       "scikit-ribo              1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_by_package.sort_values('filename', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>django</th>\n",
       "      <th>python-3.x</th>\n",
       "      <th>pandas</th>\n",
       "      <th>python-2.7</th>\n",
       "      <th>numpy</th>\n",
       "      <th>list</th>\n",
       "      <th>matplotlib</th>\n",
       "      <th>dictionary</th>\n",
       "      <th>regex</th>\n",
       "      <th>flask</th>\n",
       "      <th>...</th>\n",
       "      <th>output</th>\n",
       "      <th>urllib3</th>\n",
       "      <th>login</th>\n",
       "      <th>get</th>\n",
       "      <th>https</th>\n",
       "      <th>httprequest</th>\n",
       "      <th>http-post</th>\n",
       "      <th>http-headers</th>\n",
       "      <th>multipartform-data</th>\n",
       "      <th>grequests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>algorithm</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>870.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>arrays</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9475.0</td>\n",
       "      <td>4291.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1931.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beautifulsoup</th>\n",
       "      <td>123.0</td>\n",
       "      <td>1411.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>1188.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>421.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c++</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>class</th>\n",
       "      <td>0.0</td>\n",
       "      <td>971.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>572.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>981.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>468.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 863 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               django  python-3.x  pandas  python-2.7   numpy    list  \\\n",
       "algorithm         0.0         0.0     0.0         0.0     0.0   870.0   \n",
       "arrays            0.0         0.0     0.0         0.0  9475.0  4291.0   \n",
       "beautifulsoup   123.0      1411.0   168.0      1188.0     0.0    75.0   \n",
       "c++               0.0         0.0     0.0         0.0     0.0     0.0   \n",
       "class             0.0       971.0     0.0       572.0     0.0   981.0   \n",
       "\n",
       "               matplotlib  dictionary  regex  flask    ...      output  \\\n",
       "algorithm             0.0         0.0    0.0    0.0    ...         0.0   \n",
       "arrays                0.0      1931.0    0.0    0.0    ...         0.0   \n",
       "beautifulsoup         0.0        49.0  421.0    0.0    ...         0.0   \n",
       "c++                   0.0         0.0    0.0    0.0    ...         0.0   \n",
       "class                 0.0       468.0    0.0    0.0    ...         0.0   \n",
       "\n",
       "               urllib3  login  get  https  httprequest  http-post  \\\n",
       "algorithm          0.0    0.0  0.0    0.0          0.0        0.0   \n",
       "arrays             0.0    0.0  0.0    0.0          0.0        0.0   \n",
       "beautifulsoup      0.0    0.0  0.0    0.0          0.0        0.0   \n",
       "c++                0.0    0.0  0.0    0.0          0.0        0.0   \n",
       "class              0.0    0.0  0.0    0.0          0.0        0.0   \n",
       "\n",
       "               http-headers  multipartform-data  grequests  \n",
       "algorithm               0.0                 0.0        0.0  \n",
       "arrays                  0.0                 0.0        0.0  \n",
       "beautifulsoup           0.0                 0.0        0.0  \n",
       "c++                     0.0                 0.0        0.0  \n",
       "class                   0.0                 0.0        0.0  \n",
       "\n",
       "[5 rows x 863 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jsondf = pd.read_pickle('json_dataframe_notebook_05.pickle')\n",
    "jsondf.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the rows where `firefox` has a value higher than ten:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>django</th>\n",
       "      <th>python-3.x</th>\n",
       "      <th>pandas</th>\n",
       "      <th>python-2.7</th>\n",
       "      <th>numpy</th>\n",
       "      <th>list</th>\n",
       "      <th>matplotlib</th>\n",
       "      <th>dictionary</th>\n",
       "      <th>regex</th>\n",
       "      <th>flask</th>\n",
       "      <th>...</th>\n",
       "      <th>output</th>\n",
       "      <th>urllib3</th>\n",
       "      <th>login</th>\n",
       "      <th>get</th>\n",
       "      <th>https</th>\n",
       "      <th>httprequest</th>\n",
       "      <th>http-post</th>\n",
       "      <th>http-headers</th>\n",
       "      <th>multipartform-data</th>\n",
       "      <th>grequests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>html</th>\n",
       "      <td>4296.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5027.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>javascript</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26714.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>selenium</th>\n",
       "      <td>0.0</td>\n",
       "      <td>973.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>895.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 863 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            django  python-3.x  pandas  python-2.7  numpy  list  matplotlib  \\\n",
       "html        4296.0         0.0     0.0         0.0    0.0   0.0         0.0   \n",
       "javascript     0.0         0.0     0.0         0.0    0.0   0.0         0.0   \n",
       "selenium       0.0       973.0     0.0       895.0    0.0   0.0         0.0   \n",
       "\n",
       "            dictionary    regex  flask    ...      output  urllib3  login  \\\n",
       "html               0.0   5027.0    0.0    ...         0.0      0.0    0.0   \n",
       "javascript         0.0  26714.0    0.0    ...         0.0      0.0    0.0   \n",
       "selenium           0.0      0.0    0.0    ...         0.0      0.0    0.0   \n",
       "\n",
       "            get  https  httprequest  http-post  http-headers  \\\n",
       "html        0.0    0.0          0.0        0.0           0.0   \n",
       "javascript  0.0    0.0          0.0        0.0           0.0   \n",
       "selenium    0.0    0.0          0.0        0.0           0.0   \n",
       "\n",
       "            multipartform-data  grequests  \n",
       "html                       0.0        0.0  \n",
       "javascript                 0.0        0.0  \n",
       "selenium                   0.0        0.0  \n",
       "\n",
       "[3 rows x 863 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jsondf[jsondf['firefox'] > 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "jquery        500680.0\n",
       "css           320479.0\n",
       "html          298409.0\n",
       "javascript    298409.0\n",
       "php           209689.0\n",
       "angularjs     114721.0\n",
       "sql           100893.0\n",
       "django         86864.0\n",
       "python         86864.0\n",
       "ajax           84942.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jsondf.max().nlargest(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}