From 69e7ae5a3fae1c8769ab18a35023380c8891f040 Mon Sep 17 00:00:00 2001 From: mchenzl Date: Thu, 4 Jul 2024 15:32:12 +0100 Subject: [PATCH] added limit of 30 pages and updated readme --- README.md | 15 ++++++++++----- src/main.py | 9 +++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5868281..91bf8bc 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ -# [DEPRECATED] Glassdoor job scraper +# Glassdoor job scraper This project web scrapes the popular job listing site "Glassdoor" for information from job listings * Functions without any authentication e.g. user sign-ins/ API tokens and keys. Users simply modifies a config file to provide: - A 'base url' to scrape from, based on desired job role and country. - - A 'target job size' i.e. number of individual job listings to scrape from. + - A 'target job size' i.e. number of individual job listings to scrape from. [NOTE: Glassdoor will only show 900 jobs max (i.e. 30 jobs * 30 pages)] * Script scrapes: - Job link, role, company and job description from glassdoor job listing results. * Information collected are accessible to users in the form of an output csv. * Script has been tested and verified to be working as expected for a job with: - - A target job size of < 2000 individual listings, - - Multiple pages > 10 pages of job listing links. + - A target job size of <= 900 individual listings, + - Pages are not directly availables in the new version, but are used in the background, max of 30 pages, + - With Firefox Selenium Webdrive. ## Extracted data ![](https://github.com/kelvinxuande/glassdoor-scraper/blob/master/docs/def-3.jpg) @@ -40,4 +41,8 @@ The following gif shows how a base_url can be obtained: ## Future work -There are plans to create a data processing pipeline to analyse and visualise to generate useful insights from extracted data in the future. Feel free to collaborate and contribute to this project, or open an issue to suggest more useful features for implementation. +- Job Desciption is only a short snippet, next feature is to obtain full Job Description. +- Using Localised Version of Glassdoor website (.co.uk), next feature is to make it more dynamic. +- Next data point to scrape are Reviews. +- Refactor codebase using Requests library instead of Selenium to see if cloudfare checks can be bypassed. + diff --git a/src/main.py b/src/main.py index 874d2c8..19879e4 100644 --- a/src/main.py +++ b/src/main.py @@ -42,7 +42,12 @@ def __init__(self, configfile, baseurl, targetnum) -> None: #browser/selenium setup options = webdriver.FirefoxOptions() - options.add_argument("-headless") + options.add_argument("--devtools") + profile = webdriver.FirefoxProfile() + profile.set_preference("devtools.toolbox.selectedTool", "netmonitor") + profile.update_preferences() + options.profile = profile + #options.add_argument("-headless") driver = webdriver.Firefox(options=options) requested_url = checkURL(base_url) @@ -71,7 +76,7 @@ def __init__(self, configfile, baseurl, targetnum) -> None: # load next page if necessary and extract job details if len(list_returnedTuple) < target_num: - while len(list_returnedTuple) < min(target_num, job_count): + while len(list_returnedTuple) < min(target_num, job_count) and next_page[0]<=30: page_number, page_cursor = next_page next_page_script = fetch_next_page(user_agent, token, version, keyword, location_id, original_page_url, parameter_url_input, seo_friendly_url_input, page_cursor, page_number) response = driver.execute_script(next_page_script)