From 5ec65475803c14a870e2611691237830faf60c03 Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Mon, 27 Apr 2020 21:44:49 +0300 Subject: [PATCH 01/14] Inital commit --- smarsy/bs_helper.py | 0 tests/test_bs_helper.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 smarsy/bs_helper.py create mode 100644 tests/test_bs_helper.py diff --git a/smarsy/bs_helper.py b/smarsy/bs_helper.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py new file mode 100644 index 0000000..e69de29 From 50efe5f1eda5c22ffbe230033b81338bb6d6750c Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Mon, 27 Apr 2020 21:57:06 +0300 Subject: [PATCH 02/14] test_bshelper_instance_created added --- smarsy/bs_helper.py | 6 ++++++ tests/test_bs_helper.py | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/smarsy/bs_helper.py b/smarsy/bs_helper.py index e69de29..79406be 100644 --- a/smarsy/bs_helper.py +++ b/smarsy/bs_helper.py @@ -0,0 +1,6 @@ +class BSHelper(object): + """ + The help class for BeautifulSoup library + """ + def __init__(self, html): + self.html = html diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index e69de29..c8a5d17 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -0,0 +1,22 @@ +import unittest +import sys +import os + + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), + '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', 'smarsy'))) +# excluding following line for linter as it complains that +# from import is supposed to be at the top of the file + +from smarsy.bs_helper import BSHelper # noqa + + +class TestBSHelperInstance(unittest.TestCase): + def setUp(self): + self.html = 'some html' + self.source_page = BSHelper(self.html) + + def test_bshelper_instance_created(self): + self.assertEqual(self.source_page.html, self.html) From e1e46d248074c117708dda948bfd90b9b804214d Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Mon, 27 Apr 2020 22:02:54 +0300 Subject: [PATCH 03/14] get_bs_object func & tests added --- smarsy/bs_helper.py | 16 ++++++++++++++++ tests/test_bs_helper.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/smarsy/bs_helper.py b/smarsy/bs_helper.py index 79406be..5778146 100644 --- a/smarsy/bs_helper.py +++ b/smarsy/bs_helper.py @@ -1,6 +1,22 @@ +from bs4 import BeautifulSoup + + class BSHelper(object): """ The help class for BeautifulSoup library """ def __init__(self, html): self.html = html + + @property + def get_bs_object(self): + """ + Utility funtcion: + - Accepts html and checks its validity using BeautifulSoup library, + return BS object or False + """ + try: + soup = BeautifulSoup(self.html, 'html.parser') + except TypeError: + return False + return soup diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index c8a5d17..6a90d07 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -2,6 +2,7 @@ import sys import os +from unittest.mock import patch, PropertyMock sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) @@ -20,3 +21,18 @@ def setUp(self): def test_bshelper_instance_created(self): self.assertEqual(self.source_page.html, self.html) + + +class TestGetPageSource(unittest.TestCase): + @patch('smarsy.bs_helper.BeautifulSoup', new_callable=PropertyMock) + def test_get_bs_object_called_with_expected_html(self, mocked_soup): + html = '' + source_page = BSHelper(html) + source_page.get_bs_object + mocked_soup.assert_called_with(html, 'html.parser') + + @patch('smarsy.bs_helper.BeautifulSoup', side_effect=TypeError) + def test_get_bs_object_return_false_with_unexpected_html( + self, mocked_soup): + source_page = BSHelper(12345) + self.assertFalse(source_page.get_bs_object) From 8ac09777845dd71e765627d9b78cd92a9fc69655 Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Mon, 27 Apr 2020 22:08:03 +0300 Subject: [PATCH 04/14] TestBsSafeSelect and bs_safe_select added --- smarsy/bs_helper.py | 12 ++++++++++++ tests/test_bs_helper.py | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/smarsy/bs_helper.py b/smarsy/bs_helper.py index 5778146..a47f989 100644 --- a/smarsy/bs_helper.py +++ b/smarsy/bs_helper.py @@ -20,3 +20,15 @@ def get_bs_object(self): except TypeError: return False return soup + + def bs_safe_select(self, html, *args): + """ + Utility function used to get a content string from a + HTML and tuple of selectors. Returns False + if no object is found for the given selector + """ + for arg in args: + selectedElems = html.select_one(arg) + if selectedElems is not None: + return selectedElems + return False diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index 6a90d07..0e5a08f 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -36,3 +36,29 @@ def test_get_bs_object_return_false_with_unexpected_html( self, mocked_soup): source_page = BSHelper(12345) self.assertFalse(source_page.get_bs_object) + + +class TestBsSafeSelect(unittest.TestCase): + @patch('smarsy.bs_helper.BeautifulSoup') + def setUp(self, mocked_soup): + self.source_page = BSHelper('some html') + self.mocked_soup = mocked_soup + self.mocked_soup.select_one.return_value = 'some text' + self.selector = 'some_tag' + + def test_bs_safe_select_return_expected_text_with_single_selector(self): + actual = self.source_page.bs_safe_select(self.mocked_soup, + self.selector) + self.assertEqual(actual, 'some text') + + def test_bs_safe_select_return_expected_text_with_many_selectors(self): + selector1, selector2, selector3 = 'some_tag1', 'some_tag2', 'some_tag3' + actual = self.source_page.bs_safe_select(self.mocked_soup, selector1, + selector2, selector3) + self.assertEqual(actual, 'some text') + + def test_bs_safe_select_return_false_when_selectedElems_is_empty( + self): + self.mocked_soup.select_one.return_value = '' + self.assertFalse(self.source_page.bs_safe_select(self.mocked_soup, + self.selector)) From 42ffea0ec3c0e9ee985afe2c45385fcd5ae8e44c Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Mon, 27 Apr 2020 22:11:36 +0300 Subject: [PATCH 05/14] TestBsSafeget and bs_safe_get func added --- smarsy/bs_helper.py | 11 +++++++++++ tests/test_bs_helper.py | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/smarsy/bs_helper.py b/smarsy/bs_helper.py index a47f989..3150562 100644 --- a/smarsy/bs_helper.py +++ b/smarsy/bs_helper.py @@ -32,3 +32,14 @@ def bs_safe_select(self, html, *args): if selectedElems is not None: return selectedElems return False + + def bs_safe_get(self, html, attribute): + """ + Utility function used to get a content string from a + HTML and attribute. Returns False + if no object is found for the given selector + """ + element = html.get(attribute) + if element is not None: + return element + return False diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index 0e5a08f..8181d8f 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -62,3 +62,27 @@ def test_bs_safe_select_return_false_when_selectedElems_is_empty( self.mocked_soup.select_one.return_value = '' self.assertFalse(self.source_page.bs_safe_select(self.mocked_soup, self.selector)) + + +class TestBsSafeget(unittest.TestCase): + @patch('smarsy.bs_helper.BeautifulSoup') + def setUp(self, mocked_soup): + self.source_page = BSHelper('some html') + self.mocked_soup = mocked_soup + + def test_bs_get_called_with_expected_html_and_attribute(self): + expected_attribute = 'some attribute' + self.source_page.bs_safe_get(self.mocked_soup, expected_attribute) + self.mocked_soup.get.assert_called_with(expected_attribute) + + def test_bs_safe_get_return_false_when_element_is_empty( + self): + self.mocked_soup.get.return_value = '' + self.assertFalse(self.source_page.bs_safe_get(self.mocked_soup, + 'some attribute')) + + def test_bs_safe_get_return_expected_text(self): + self.mocked_soup.get.return_value = 'some text' + actual = self.source_page.bs_safe_get(self.mocked_soup, + 'some attribute') + self.assertEqual(actual, 'some text') From ec0dcb593664aac660618b9fb045c77c1181f05a Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Tue, 28 Apr 2020 11:38:53 +0300 Subject: [PATCH 06/14] get_bs_object renamed to bs_object --- smarsy/bs_helper.py | 2 +- tests/test_bs_helper.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/smarsy/bs_helper.py b/smarsy/bs_helper.py index 3150562..be8497f 100644 --- a/smarsy/bs_helper.py +++ b/smarsy/bs_helper.py @@ -9,7 +9,7 @@ def __init__(self, html): self.html = html @property - def get_bs_object(self): + def bs_object(self): """ Utility funtcion: - Accepts html and checks its validity using BeautifulSoup library, diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index 8181d8f..9e688b6 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -25,17 +25,17 @@ def test_bshelper_instance_created(self): class TestGetPageSource(unittest.TestCase): @patch('smarsy.bs_helper.BeautifulSoup', new_callable=PropertyMock) - def test_get_bs_object_called_with_expected_html(self, mocked_soup): + def test_bs_object_called_with_expected_html(self, mocked_soup): html = '' source_page = BSHelper(html) - source_page.get_bs_object + source_page.bs_object mocked_soup.assert_called_with(html, 'html.parser') @patch('smarsy.bs_helper.BeautifulSoup', side_effect=TypeError) - def test_get_bs_object_return_false_with_unexpected_html( + def test_bs_object_return_false_with_unexpected_html( self, mocked_soup): source_page = BSHelper(12345) - self.assertFalse(source_page.get_bs_object) + self.assertFalse(source_page.bs_object) class TestBsSafeSelect(unittest.TestCase): From ca84c3e2b1917aadf682dbd69566ecc01259743a Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Tue, 28 Apr 2020 11:41:07 +0300 Subject: [PATCH 07/14] Deleted setUp in TestBSHelperInstance --- tests/test_bs_helper.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index 9e688b6..eafc29c 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -15,12 +15,10 @@ class TestBSHelperInstance(unittest.TestCase): - def setUp(self): - self.html = 'some html' - self.source_page = BSHelper(self.html) - def test_bshelper_instance_created(self): - self.assertEqual(self.source_page.html, self.html) + html = 'some html' + source_page = BSHelper(html) + self.assertEqual(source_page.html, html) class TestGetPageSource(unittest.TestCase): From 4234720c908be6cf0a78689a5df76300e62688d2 Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Tue, 28 Apr 2020 11:44:31 +0300 Subject: [PATCH 08/14] Testclasses for methods renamed --- tests/test_bs_helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index eafc29c..fd14ba1 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -21,7 +21,7 @@ def test_bshelper_instance_created(self): self.assertEqual(source_page.html, html) -class TestGetPageSource(unittest.TestCase): +class Test_bs_object(unittest.TestCase): @patch('smarsy.bs_helper.BeautifulSoup', new_callable=PropertyMock) def test_bs_object_called_with_expected_html(self, mocked_soup): html = '' @@ -36,7 +36,7 @@ def test_bs_object_return_false_with_unexpected_html( self.assertFalse(source_page.bs_object) -class TestBsSafeSelect(unittest.TestCase): +class Test_bs_safe_select(unittest.TestCase): @patch('smarsy.bs_helper.BeautifulSoup') def setUp(self, mocked_soup): self.source_page = BSHelper('some html') @@ -62,7 +62,7 @@ def test_bs_safe_select_return_false_when_selectedElems_is_empty( self.selector)) -class TestBsSafeget(unittest.TestCase): +class Test_bs_safe_get(unittest.TestCase): @patch('smarsy.bs_helper.BeautifulSoup') def setUp(self, mocked_soup): self.source_page = BSHelper('some html') From 6342d7a8b3de2ccdaecd36adf984b08b5366522f Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Tue, 28 Apr 2020 12:15:30 +0300 Subject: [PATCH 09/14] Test_bs_object refactored --- tests/test_bs_helper.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index fd14ba1..ed377f7 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -25,8 +25,7 @@ class Test_bs_object(unittest.TestCase): @patch('smarsy.bs_helper.BeautifulSoup', new_callable=PropertyMock) def test_bs_object_called_with_expected_html(self, mocked_soup): html = '' - source_page = BSHelper(html) - source_page.bs_object + source_page = BSHelper(html).bs_object mocked_soup.assert_called_with(html, 'html.parser') @patch('smarsy.bs_helper.BeautifulSoup', side_effect=TypeError) @@ -43,6 +42,7 @@ def setUp(self, mocked_soup): self.mocked_soup = mocked_soup self.mocked_soup.select_one.return_value = 'some text' self.selector = 'some_tag' + self.expected_text = 'some_tag1', 'some_tag2', 'some_tag3' def test_bs_safe_select_return_expected_text_with_single_selector(self): actual = self.source_page.bs_safe_select(self.mocked_soup, @@ -50,12 +50,11 @@ def test_bs_safe_select_return_expected_text_with_single_selector(self): self.assertEqual(actual, 'some text') def test_bs_safe_select_return_expected_text_with_many_selectors(self): - selector1, selector2, selector3 = 'some_tag1', 'some_tag2', 'some_tag3' - actual = self.source_page.bs_safe_select(self.mocked_soup, selector1, - selector2, selector3) + actual = self.source_page.bs_safe_select(self.mocked_soup, + self.expected_text) self.assertEqual(actual, 'some text') - def test_bs_safe_select_return_false_when_selectedElems_is_empty( + def test_bs_safe_select_return_false_when_no_object_is_found( self): self.mocked_soup.select_one.return_value = '' self.assertFalse(self.source_page.bs_safe_select(self.mocked_soup, From 6e6b9875fff3a7de1cc8462cb1a502214f7a5937 Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Tue, 28 Apr 2020 14:23:05 +0300 Subject: [PATCH 10/14] test_bs_safe_select_return_expected_text_with_many_selectors refactored --- tests/test_bs_helper.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index ed377f7..4beba1f 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -42,7 +42,8 @@ def setUp(self, mocked_soup): self.mocked_soup = mocked_soup self.mocked_soup.select_one.return_value = 'some text' self.selector = 'some_tag' - self.expected_text = 'some_tag1', 'some_tag2', 'some_tag3' + self.selectors = 'some_tag1', 'some_tag2', 'some_tag3' + self.select_one_values = ('some text1', 'some text2', 'some text3') def test_bs_safe_select_return_expected_text_with_single_selector(self): actual = self.source_page.bs_safe_select(self.mocked_soup, @@ -50,9 +51,13 @@ def test_bs_safe_select_return_expected_text_with_single_selector(self): self.assertEqual(actual, 'some text') def test_bs_safe_select_return_expected_text_with_many_selectors(self): + select_one = None + for select_one_value in self.select_one_values: + select_one = select_one_value + self.mocked_soup.select_one.return_value = select_one actual = self.source_page.bs_safe_select(self.mocked_soup, - self.expected_text) - self.assertEqual(actual, 'some text') + self.selectors) + self.assertEqual(actual, select_one) def test_bs_safe_select_return_false_when_no_object_is_found( self): From 3faccda0f5aed777475bbb9daff345185f70022e Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Tue, 28 Apr 2020 20:03:11 +0300 Subject: [PATCH 11/14] self.expected added in Test_bs_safe_select& expected_text varin test_bs_safe_get_return_false_ --- tests/test_bs_helper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index 4beba1f..b3b1a0d 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -44,11 +44,12 @@ def setUp(self, mocked_soup): self.selector = 'some_tag' self.selectors = 'some_tag1', 'some_tag2', 'some_tag3' self.select_one_values = ('some text1', 'some text2', 'some text3') + self.expected = 'some text' def test_bs_safe_select_return_expected_text_with_single_selector(self): actual = self.source_page.bs_safe_select(self.mocked_soup, self.selector) - self.assertEqual(actual, 'some text') + self.assertEqual(actual, self.expected) def test_bs_safe_select_return_expected_text_with_many_selectors(self): select_one = None @@ -85,6 +86,7 @@ def test_bs_safe_get_return_false_when_element_is_empty( def test_bs_safe_get_return_expected_text(self): self.mocked_soup.get.return_value = 'some text' + expected_text = 'some text' actual = self.source_page.bs_safe_get(self.mocked_soup, 'some attribute') - self.assertEqual(actual, 'some text') + self.assertEqual(actual, expected_text) From 34358c89aabd11049f5197b6beac836027bdc62d Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Wed, 29 Apr 2020 15:52:47 +0300 Subject: [PATCH 12/14] expected_text to Test_bs_safe_get setUp --- tests/test_bs_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index b3b1a0d..20fa9a2 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -72,6 +72,7 @@ class Test_bs_safe_get(unittest.TestCase): def setUp(self, mocked_soup): self.source_page = BSHelper('some html') self.mocked_soup = mocked_soup + self.expected_text = 'some text' def test_bs_get_called_with_expected_html_and_attribute(self): expected_attribute = 'some attribute' @@ -86,7 +87,6 @@ def test_bs_safe_get_return_false_when_element_is_empty( def test_bs_safe_get_return_expected_text(self): self.mocked_soup.get.return_value = 'some text' - expected_text = 'some text' actual = self.source_page.bs_safe_get(self.mocked_soup, 'some attribute') - self.assertEqual(actual, expected_text) + self.assertEqual(actual, self.expected_text) From b789612b89a71453c45d54fe2e8f2c345c9b0b84 Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Wed, 29 Apr 2020 15:54:26 +0300 Subject: [PATCH 13/14] expected_attribute to setUp --- tests/test_bs_helper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index 20fa9a2..d73f926 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -73,20 +73,20 @@ def setUp(self, mocked_soup): self.source_page = BSHelper('some html') self.mocked_soup = mocked_soup self.expected_text = 'some text' + self.expected_attribute = 'some attribute' def test_bs_get_called_with_expected_html_and_attribute(self): - expected_attribute = 'some attribute' - self.source_page.bs_safe_get(self.mocked_soup, expected_attribute) - self.mocked_soup.get.assert_called_with(expected_attribute) + self.source_page.bs_safe_get(self.mocked_soup, self.expected_attribute) + self.mocked_soup.get.assert_called_with(self.expected_attribute) def test_bs_safe_get_return_false_when_element_is_empty( self): self.mocked_soup.get.return_value = '' self.assertFalse(self.source_page.bs_safe_get(self.mocked_soup, - 'some attribute')) + self.expected_attribute)) def test_bs_safe_get_return_expected_text(self): self.mocked_soup.get.return_value = 'some text' actual = self.source_page.bs_safe_get(self.mocked_soup, - 'some attribute') + self.expected_attribute) self.assertEqual(actual, self.expected_text) From 06a5df10381763b8e9642d1a2d5f0c54d97bc4f1 Mon Sep 17 00:00:00 2001 From: MaxxxZu Date: Wed, 29 Apr 2020 16:38:21 +0300 Subject: [PATCH 14/14] mocked_soup.get.return_value ti setUp --- tests/test_bs_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bs_helper.py b/tests/test_bs_helper.py index d73f926..9c82234 100644 --- a/tests/test_bs_helper.py +++ b/tests/test_bs_helper.py @@ -74,6 +74,7 @@ def setUp(self, mocked_soup): self.mocked_soup = mocked_soup self.expected_text = 'some text' self.expected_attribute = 'some attribute' + self.mocked_soup.get.return_value = 'some text' def test_bs_get_called_with_expected_html_and_attribute(self): self.source_page.bs_safe_get(self.mocked_soup, self.expected_attribute) @@ -86,7 +87,6 @@ def test_bs_safe_get_return_false_when_element_is_empty( self.expected_attribute)) def test_bs_safe_get_return_expected_text(self): - self.mocked_soup.get.return_value = 'some text' actual = self.source_page.bs_safe_get(self.mocked_soup, self.expected_attribute) self.assertEqual(actual, self.expected_text)