Scraping-Code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b37baa98",
   "metadata": {},
   "source": [
    "### Loading Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e26335ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "423c73aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "# requst page\n",
    "\n",
    "page = requests.get(\"https://wuzzuf.net/search/jobs/?a=navbl&filters%5Bcountry%5D%5B0%5D=Egypt&q=analysis\")\n",
    "print(page)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdf12a84",
   "metadata": {},
   "source": [
    "- Response 200 Means success request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5b446c1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get page content\n",
    "\n",
    "src = page.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c8ed1dbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse content of page\n",
    "\n",
    "soup = BeautifulSoup(src, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "803a6447",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all jobs\n",
    "\n",
    "jobs = soup.find_all(\"div\", {\"class\" : \"css-pkv5jc\"})\n",
    "len(jobs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "537137a2",
   "metadata": {},
   "source": [
    "### Testing whether data correct or not "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "30bcc0b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Financial Analysis'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get job name\n",
    "\n",
    "job_name = jobs[0].find_all(\"h2\", {\"class\" : \"css-m604qf\"})[0].text\n",
    "job_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2f32cdf0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'World Food Program - Other locations -'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get campany name\n",
    "\n",
    "campany_name = jobs[1].find_all(\"a\", {\"class\" : \"css-17s97q8\"})[0].text\n",
    "campany_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3e90ff5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Cairo, Egypt '"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get campany location\n",
    "\n",
    "campany_location = jobs[3].find_all(\"span\", {\"class\" : \"css-5wys0k\"})[0].text\n",
    "campany_location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3ad81911",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Full Time'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get job type\n",
    "\n",
    "job_type = jobs[14].find_all(\"span\", {\"class\" : \"css-1ve4b75 eoyjyou0\"})[0].text\n",
    "job_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bd848e24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Senior Management'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get Experience level\n",
    "\n",
    "exp_level = jobs[13].find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[0]\n",
    "exp_level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c06a840c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5 - 7 Yrs of Exp'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get Experience year\n",
    "\n",
    "exp_year = jobs[13].find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[1]\n",
    "exp_year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "970bc5a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Accounting/Finance,Financial Analysis,Financial Management,Investment'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get job skills\n",
    "\n",
    "skills = jobs[13].find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[2:]\n",
    "\",\".join(skills)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eae28330",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job title</th>\n",
       "      <th>Company name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job type</th>\n",
       "      <th>Exp level</th>\n",
       "      <th>Exp years</th>\n",
       "      <th>Skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Job title, Company name, Location, Job type, Exp level, Exp years, Skills]\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create Data Frame\n",
    "\n",
    "data = pd.DataFrame(columns = [\"Job title\", \"Company name\", \"Location\", \"Job type\", \"Exp level\", \"Exp years\", \"Skills\"])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7959a68",
   "metadata": {},
   "source": [
    "- Data frame still empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "059f047e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add data to Data frame\n",
    "\n",
    "for job in jobs:\n",
    "    job_name =  job.find_all(\"h2\", {\"class\" : \"css-m604qf\"})[0].text\n",
    "    \n",
    "    company_name = job.find_all(\"a\", {\"class\" : \"css-17s97q8\"})[0].text\n",
    "    \n",
    "    location = job.find_all(\"span\", {\"class\" : \"css-5wys0k\"})[0].text\n",
    "    \n",
    "    job_type =  job.find_all(\"span\", {\"class\" : \"css-1ve4b75 eoyjyou0\"})[0].text\n",
    "    \n",
    "    exp_level = job.find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[0]\n",
    "    \n",
    "    exp_years = job.find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[1]\n",
    "    \n",
    "    skills = \",\".join(job.find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[2:])\n",
    "    \n",
    "    data.loc[len(data.index)] = [job_name, company_name, location, job_type, exp_level, exp_years, skills]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0dbd6ca1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job title</th>\n",
       "      <th>Company name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job type</th>\n",
       "      <th>Exp level</th>\n",
       "      <th>Exp years</th>\n",
       "      <th>Skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Financial Analysis</td>\n",
       "      <td>Care Services EG -</td>\n",
       "      <td>Abbassia, Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>2 - 4 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Analyst/Research,Accounting...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Vulnerability Analysis &amp; Mapping Officer (Econ...</td>\n",
       "      <td>World Food Program - Other locations -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>IT/Software Development</td>\n",
       "      <td>Project/Program Management,Engineering - Telec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Financial Analysis &amp; Planning Manager</td>\n",
       "      <td>Kinetik People -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Accounting/Finance</td>\n",
       "      <td>Operations/Management,Analyst/Research,Financi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Financial Planning &amp; Analysis Lead</td>\n",
       "      <td>Novartis ÖSterreich -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Accounting/Finance</td>\n",
       "      <td>Operations/Management,Analyst/Research,Finance...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Analysis and Reporting Manager</td>\n",
       "      <td>Antal International - Egypt -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Accounting/Finance</td>\n",
       "      <td>Operations/Management,Analyst/Research,Finance...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Senior Analyst,Financial Planning and Analysis</td>\n",
       "      <td>S. C. Johnson &amp; Son, Inc. -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Accounting/Finance</td>\n",
       "      <td>Business Development,Analyst/Research,Finance,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Financial Planning And Analysis Supervisor</td>\n",
       "      <td>Confidential -</td>\n",
       "      <td>Mohandessin, Giza, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>4 - 7 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Analyst/Research,Accounting...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>data analysis</td>\n",
       "      <td>Care Services EG -</td>\n",
       "      <td>Abbassia, Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Entry Level</td>\n",
       "      <td>2 - 4 Yrs of Exp</td>\n",
       "      <td>IT/Software Development,Analyst/Research,Analy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Financial Planning &amp; Analysis Assistant Manager</td>\n",
       "      <td>Network International -Egypt -</td>\n",
       "      <td>New Cairo, Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>10 - 11 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Banking,Financial Analysis,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Senior Financial Planning and Analysis (FP&amp;A) ...</td>\n",
       "      <td>ARPUPLUS -</td>\n",
       "      <td>Giza, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>3 - 5 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Analyst/Research,Accounting...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>business analysis</td>\n",
       "      <td>B-Solution -</td>\n",
       "      <td>6th of October, Giza, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Senior Management</td>\n",
       "      <td>1 - 20 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,IT/Software Development,Ana...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Senior Financial Planning &amp; Analysis</td>\n",
       "      <td>Confidential -</td>\n",
       "      <td>Mohandessin, Giza, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>4 - 7 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Finance,Financial Analysis,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Budgeting &amp; Financial Analysis Manager</td>\n",
       "      <td>Confidential -</td>\n",
       "      <td>Gharbia, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Manager</td>\n",
       "      <td>8 - 10 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Analyst/Research,analytical...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Financial analysis accountant and balancers</td>\n",
       "      <td>Royal Investment -</td>\n",
       "      <td>6th of October, Giza, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Senior Management</td>\n",
       "      <td>5 - 7 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Financial Analysis,Financia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Associate Manager - Financial Planning &amp; Analysis</td>\n",
       "      <td>S. C. Johnson &amp; Son, Inc. -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>IT/Software Development</td>\n",
       "      <td>Project/Program Management,Engineering - Telec...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Job title  \\\n",
       "0                                  Financial Analysis   \n",
       "1   Vulnerability Analysis & Mapping Officer (Econ...   \n",
       "2               Financial Analysis & Planning Manager   \n",
       "3                  Financial Planning & Analysis Lead   \n",
       "4                      Analysis and Reporting Manager   \n",
       "5      Senior Analyst,Financial Planning and Analysis   \n",
       "6          Financial Planning And Analysis Supervisor   \n",
       "7                                       data analysis   \n",
       "8     Financial Planning & Analysis Assistant Manager   \n",
       "9   Senior Financial Planning and Analysis (FP&A) ...   \n",
       "10                                  business analysis   \n",
       "11               Senior Financial Planning & Analysis   \n",
       "12             Budgeting & Financial Analysis Manager   \n",
       "13        Financial analysis accountant and balancers   \n",
       "14  Associate Manager - Financial Planning & Analysis   \n",
       "\n",
       "                              Company name                      Location  \\\n",
       "0                       Care Services EG -       Abbassia, Cairo, Egypt    \n",
       "1   World Food Program - Other locations -                 Cairo, Egypt    \n",
       "2                         Kinetik People -                 Cairo, Egypt    \n",
       "3                    Novartis ÖSterreich -                 Cairo, Egypt    \n",
       "4            Antal International - Egypt -                 Cairo, Egypt    \n",
       "5              S. C. Johnson & Son, Inc. -                 Cairo, Egypt    \n",
       "6                           Confidential -     Mohandessin, Giza, Egypt    \n",
       "7                       Care Services EG -       Abbassia, Cairo, Egypt    \n",
       "8           Network International -Egypt -      New Cairo, Cairo, Egypt    \n",
       "9                               ARPUPLUS -                  Giza, Egypt    \n",
       "10                            B-Solution -  6th of October, Giza, Egypt    \n",
       "11                          Confidential -     Mohandessin, Giza, Egypt    \n",
       "12                          Confidential -               Gharbia, Egypt    \n",
       "13                      Royal Investment -  6th of October, Giza, Egypt    \n",
       "14             S. C. Johnson & Son, Inc. -                 Cairo, Egypt    \n",
       "\n",
       "     Job type          Exp level                Exp years  \\\n",
       "0   Full Time        Experienced         2 - 4 Yrs of Exp   \n",
       "1   Full Time      Not specified  IT/Software Development   \n",
       "2   Full Time      Not specified       Accounting/Finance   \n",
       "3   Full Time      Not specified       Accounting/Finance   \n",
       "4   Full Time      Not specified       Accounting/Finance   \n",
       "5   Full Time      Not specified       Accounting/Finance   \n",
       "6   Full Time        Experienced         4 - 7 Yrs of Exp   \n",
       "7   Full Time        Entry Level         2 - 4 Yrs of Exp   \n",
       "8   Full Time        Experienced       10 - 11 Yrs of Exp   \n",
       "9   Full Time        Experienced         3 - 5 Yrs of Exp   \n",
       "10  Full Time  Senior Management        1 - 20 Yrs of Exp   \n",
       "11  Full Time        Experienced         4 - 7 Yrs of Exp   \n",
       "12  Full Time            Manager        8 - 10 Yrs of Exp   \n",
       "13  Full Time  Senior Management         5 - 7 Yrs of Exp   \n",
       "14  Full Time      Not specified  IT/Software Development   \n",
       "\n",
       "                                               Skills  \n",
       "0   Accounting/Finance,Analyst/Research,Accounting...  \n",
       "1   Project/Program Management,Engineering - Telec...  \n",
       "2   Operations/Management,Analyst/Research,Financi...  \n",
       "3   Operations/Management,Analyst/Research,Finance...  \n",
       "4   Operations/Management,Analyst/Research,Finance...  \n",
       "5   Business Development,Analyst/Research,Finance,...  \n",
       "6   Accounting/Finance,Analyst/Research,Accounting...  \n",
       "7   IT/Software Development,Analyst/Research,Analy...  \n",
       "8   Accounting/Finance,Banking,Financial Analysis,...  \n",
       "9   Accounting/Finance,Analyst/Research,Accounting...  \n",
       "10  Accounting/Finance,IT/Software Development,Ana...  \n",
       "11  Accounting/Finance,Finance,Financial Analysis,...  \n",
       "12  Accounting/Finance,Analyst/Research,analytical...  \n",
       "13  Accounting/Finance,Financial Analysis,Financia...  \n",
       "14  Project/Program Management,Engineering - Telec...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show data\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58c91481",
   "metadata": {},
   "source": [
    "### From above :\n",
    "- Request page and get Html text.\n",
    "- Convert byte code to understandable Html text that can be read.\n",
    "- Ensure that the extracted data and tags name are correct before adding the data to the data frame.\n",
    "- Create data frame call `data` to Save extracted data it in.\n",
    "\n",
    "#### `Data Frame contains only first page of website, but i need to extract data from all pages That appeared in result.`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25267ff0",
   "metadata": {},
   "source": [
    "### Extract data from all pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "47a1b4c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 : Requests still work\n",
      "20 : Requests still work\n",
      "40 : Requests still work\n",
      "60 : Requests still work\n",
      "80 : Requests still work\n",
      "100 : Requests still work\n"
     ]
    }
   ],
   "source": [
    "for pageIndex in range(108):\n",
    "    # request page base index\n",
    "    page = requests.get(f\"https://wuzzuf.net/search/jobs/?a=navbl&filters%5Bcountry%5D%5B0%5D=Egypt&q=analysis&start={pageIndex}\")\n",
    "    \n",
    "    # get page content\n",
    "    src = page.content\n",
    "    \n",
    "    # parse content of page\n",
    "    soup = BeautifulSoup(src, \"lxml\")\n",
    "    \n",
    "    # get all jobs in page\n",
    "    jobs = soup.find_all(\"div\", {\"class\" : \"css-pkv5jc\"})\n",
    "    \n",
    "    # loop of each page to extracte data\n",
    "    for job in jobs:\n",
    "        job_name =  job.find_all(\"h2\", {\"class\" : \"css-m604qf\"})[0].text\n",
    "\n",
    "        company_name = job.find_all(\"a\", {\"class\" : \"css-17s97q8\"})[0].text\n",
    "\n",
    "        location = job.find_all(\"span\", {\"class\" : \"css-5wys0k\"})[0].text\n",
    "\n",
    "        job_type =  job.find_all(\"span\", {\"class\" : \"css-1ve4b75 eoyjyou0\"})[0].text\n",
    "\n",
    "        exp_level = job.find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[0]\n",
    "\n",
    "        exp_years = job.find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[1]\n",
    "\n",
    "        skills = \",\".join(job.find_all(\"div\", {\"class\" : \"css-1lh32fc\"})[0].next_sibling.text.split(\" · \")[2:])\n",
    "        \n",
    "        # Add each iteration of inner loop to data frame as row\n",
    "        data.loc[len(data.index)] = [job_name, company_name, location, job_type, exp_level, exp_years, skills]\n",
    "        \n",
    "    # Add condition to check requests still work or not\n",
    "    if pageIndex % 20 == 0:\n",
    "        print(f\"Succeeded Scraping {pageIndex} pages, Requests still work\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e9360355",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job title</th>\n",
       "      <th>Company name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job type</th>\n",
       "      <th>Exp level</th>\n",
       "      <th>Exp years</th>\n",
       "      <th>Skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Financial Analysis</td>\n",
       "      <td>Care Services EG -</td>\n",
       "      <td>Abbassia, Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>2 - 4 Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Analyst/Research,Accounting...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Vulnerability Analysis &amp; Mapping Officer (Econ...</td>\n",
       "      <td>World Food Program - Other locations -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>IT/Software Development</td>\n",
       "      <td>Project/Program Management,Engineering - Telec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Financial Analysis &amp; Planning Manager</td>\n",
       "      <td>Kinetik People -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Accounting/Finance</td>\n",
       "      <td>Operations/Management,Analyst/Research,Financi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Financial Planning &amp; Analysis Lead</td>\n",
       "      <td>Novartis ÖSterreich -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Accounting/Finance</td>\n",
       "      <td>Operations/Management,Analyst/Research,Finance...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Analysis and Reporting Manager</td>\n",
       "      <td>Antal International - Egypt -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Not specified</td>\n",
       "      <td>Accounting/Finance</td>\n",
       "      <td>Operations/Management,Analyst/Research,Finance...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1612</th>\n",
       "      <td>Senior Accountant</td>\n",
       "      <td>Nile University -</td>\n",
       "      <td>Sheikh Zayed, Giza, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>5+ Yrs of Exp</td>\n",
       "      <td>Accounting/Finance,Accounting,ERP,Finance,acco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1613</th>\n",
       "      <td>Senior Social Media Specialist</td>\n",
       "      <td>Twerlo -</td>\n",
       "      <td>Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>3 - 8 Yrs of Exp</td>\n",
       "      <td>Creative/Design/Art,Writing/Editorial,Marketin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1614</th>\n",
       "      <td>Production Manager - cosmetics</td>\n",
       "      <td>Elle cosmetics -</td>\n",
       "      <td>6th of October, Giza, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Manager</td>\n",
       "      <td>5 - 9 Yrs of Exp</td>\n",
       "      <td>Manufacturing/Production,Engineering - Mechani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1615</th>\n",
       "      <td>Marketing Executive</td>\n",
       "      <td>Micro Engineering -</td>\n",
       "      <td>Heliopolis, Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>2 - 4 Yrs of Exp</td>\n",
       "      <td>Marketing/PR/Advertising,Media/Journalism/Publ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1616</th>\n",
       "      <td>Technical Studies and Proposal Engineer</td>\n",
       "      <td>EL- Gammal industrial  System -</td>\n",
       "      <td>Maadi, Cairo, Egypt</td>\n",
       "      <td>Full Time</td>\n",
       "      <td>Experienced</td>\n",
       "      <td>2 - 3 Yrs of Exp</td>\n",
       "      <td>Engineering - Mechanical/Electrical,technical,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1617 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Job title  \\\n",
       "0                                    Financial Analysis   \n",
       "1     Vulnerability Analysis & Mapping Officer (Econ...   \n",
       "2                 Financial Analysis & Planning Manager   \n",
       "3                    Financial Planning & Analysis Lead   \n",
       "4                        Analysis and Reporting Manager   \n",
       "...                                                 ...   \n",
       "1612                                  Senior Accountant   \n",
       "1613                     Senior Social Media Specialist   \n",
       "1614                     Production Manager - cosmetics   \n",
       "1615                                Marketing Executive   \n",
       "1616            Technical Studies and Proposal Engineer   \n",
       "\n",
       "                                Company name                      Location  \\\n",
       "0                         Care Services EG -       Abbassia, Cairo, Egypt    \n",
       "1     World Food Program - Other locations -                 Cairo, Egypt    \n",
       "2                           Kinetik People -                 Cairo, Egypt    \n",
       "3                      Novartis ÖSterreich -                 Cairo, Egypt    \n",
       "4              Antal International - Egypt -                 Cairo, Egypt    \n",
       "...                                      ...                           ...   \n",
       "1612                       Nile University -    Sheikh Zayed, Giza, Egypt    \n",
       "1613                                Twerlo -                 Cairo, Egypt    \n",
       "1614                        Elle cosmetics -  6th of October, Giza, Egypt    \n",
       "1615                     Micro Engineering -     Heliopolis, Cairo, Egypt    \n",
       "1616         EL- Gammal industrial  System -          Maadi, Cairo, Egypt    \n",
       "\n",
       "       Job type      Exp level                Exp years  \\\n",
       "0     Full Time    Experienced         2 - 4 Yrs of Exp   \n",
       "1     Full Time  Not specified  IT/Software Development   \n",
       "2     Full Time  Not specified       Accounting/Finance   \n",
       "3     Full Time  Not specified       Accounting/Finance   \n",
       "4     Full Time  Not specified       Accounting/Finance   \n",
       "...         ...            ...                      ...   \n",
       "1612  Full Time    Experienced            5+ Yrs of Exp   \n",
       "1613  Full Time    Experienced         3 - 8 Yrs of Exp   \n",
       "1614  Full Time        Manager         5 - 9 Yrs of Exp   \n",
       "1615  Full Time    Experienced         2 - 4 Yrs of Exp   \n",
       "1616  Full Time    Experienced         2 - 3 Yrs of Exp   \n",
       "\n",
       "                                                 Skills  \n",
       "0     Accounting/Finance,Analyst/Research,Accounting...  \n",
       "1     Project/Program Management,Engineering - Telec...  \n",
       "2     Operations/Management,Analyst/Research,Financi...  \n",
       "3     Operations/Management,Analyst/Research,Finance...  \n",
       "4     Operations/Management,Analyst/Research,Finance...  \n",
       "...                                                 ...  \n",
       "1612  Accounting/Finance,Accounting,ERP,Finance,acco...  \n",
       "1613  Creative/Design/Art,Writing/Editorial,Marketin...  \n",
       "1614  Manufacturing/Production,Engineering - Mechani...  \n",
       "1615  Marketing/PR/Advertising,Media/Journalism/Publ...  \n",
       "1616  Engineering - Mechanical/Electrical,technical,...  \n",
       "\n",
       "[1617 rows x 7 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "981ba7b8",
   "metadata": {},
   "source": [
    "### - `Finished extract data from all pages That appeared in result.`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "56c544ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data frame to csv file\n",
    "\n",
    "data.to_csv(\"Wuzzuf Jobs.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}