From eabf7df0eba1a7edf25712c3272d171d5aeb11ea Mon Sep 17 00:00:00 2001 From: Victor Ruiz Date: Tue, 20 Aug 2024 10:37:30 +0200 Subject: [PATCH 1/3] Fix linting errors --- src/scrapy_settings_log/__init__.py | 16 +++++++---- tests/test_code.py | 44 ++++++++++++++++------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/scrapy_settings_log/__init__.py b/src/scrapy_settings_log/__init__.py index fc6f3c5..07ba54a 100644 --- a/src/scrapy_settings_log/__init__.py +++ b/src/scrapy_settings_log/__init__.py @@ -10,9 +10,9 @@ logger = logging.getLogger(__name__) DEFAULT_REGEXES = [ - ".*(?i)(api[\W_]*key).*", # apikey and variations e.g: shub_apikey or SC_APIKEY - ".*(?i)(AWS[\W_]*(SECRET[\W_]*)?(ACCESS)?[\W_]*(KEY|ACCESS[\W_]*KEY))", # AWS_SECRET_ACCESS_KEY and variations - ".*(?i)([\W_]*password[\W_]*).*" # password word + r".*(?i)(api[\W_]*key).*", # apikey and variations e.g: shub_apikey or SC_APIKEY + r".*(?i)(AWS[\W_]*(SECRET[\W_]*)?(ACCESS)?[\W_]*(KEY|ACCESS[\W_]*KEY))", # AWS_SECRET_ACCESS_KEY and variations + r".*(?i)([\W_]*password[\W_]*).*", # password word ] @@ -58,9 +58,15 @@ def spider_closed(self, spider): if regex is not None: settings = {k: v for k, v in settings.items() if re.search(regex, k)} if spider.settings.getbool("MASKED_SENSITIVE_SETTINGS_ENABLED", True): - regex_list = spider.settings.getlist("MASKED_SENSITIVE_SETTINGS_REGEX_LIST", DEFAULT_REGEXES) + regex_list = spider.settings.getlist( + "MASKED_SENSITIVE_SETTINGS_REGEX_LIST", DEFAULT_REGEXES + ) for reg in regex_list: - updated_settings = {k: '**********' if v else v for k, v in settings.items() if re.match(reg, k)} + updated_settings = { + k: "**********" if v else v + for k, v in settings.items() + if re.match(reg, k) + } settings = {**settings, **updated_settings} self.output_settings(settings, spider) diff --git a/tests/test_code.py b/tests/test_code.py index 31bdfa7..dd3c3e0 100644 --- a/tests/test_code.py +++ b/tests/test_code.py @@ -108,9 +108,9 @@ class CustomClass: def test_log_all_should_not_return_apikey_value_by_default(caplog): settings = { "SETTINGS_LOGGING_ENABLED": True, - "SHUB_APIKEY": 'apikey_value1', - "shub_apikey": 'apikey_value2', - "api_key": 'apikey_value3', + "SHUB_APIKEY": "apikey_value1", + "shub_apikey": "apikey_value2", + "api_key": "apikey_value3", } spider = MockSpider(settings) @@ -121,13 +121,15 @@ def test_log_all_should_not_return_apikey_value_by_default(caplog): assert '"SHUB_APIKEY": "**********"' in caplog.text assert '"shub_apikey": "**********"' in caplog.text assert '"api_key": "**********"' in caplog.text - assert 'apikey_value' not in caplog.text + assert "apikey_value" not in caplog.text -def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED_is_false(caplog): +def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED_is_false( + caplog, +): settings = { "SETTINGS_LOGGING_ENABLED": True, - "APIKEY": 'apikey_value', + "APIKEY": "apikey_value", "MASKED_SENSITIVE_SETTINGS_ENABLED": False, } @@ -142,11 +144,11 @@ def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED def test_log_all_should_not_return_aws_secret_key_value_by_default(caplog): settings = { "SETTINGS_LOGGING_ENABLED": True, - "AWS_SECRET_ACCESS_KEY": 'secret_value1', - "aws_secret_access_key": 'secret_value2', - "aws_access_key": 'secret_value2', - "AWS_SECRET_KEY": 'secret_value2', - "aws_secret_key": 'secret_value2', + "AWS_SECRET_ACCESS_KEY": "secret_value1", + "aws_secret_access_key": "secret_value2", + "aws_access_key": "secret_value2", + "AWS_SECRET_KEY": "secret_value2", + "aws_secret_key": "secret_value2", } spider = MockSpider(settings) @@ -159,14 +161,14 @@ def test_log_all_should_not_return_aws_secret_key_value_by_default(caplog): assert '"aws_access_key": "**********"' in caplog.text assert '"AWS_SECRET_KEY": "**********"' in caplog.text assert '"aws_secret_key": "**********"' in caplog.text - assert 'secret_value' not in caplog.text + assert "secret_value" not in caplog.text def test_log_all_should_not_return_password_value_by_default(caplog): settings = { "SETTINGS_LOGGING_ENABLED": True, - "test_password": 'secret_value1', - "PASSWORD_TEST": 'secret_value2', + "test_password": "secret_value1", + "PASSWORD_TEST": "secret_value2", } spider = MockSpider(settings) @@ -176,15 +178,17 @@ def test_log_all_should_not_return_password_value_by_default(caplog): assert '"test_password": "**********"' in caplog.text assert '"PASSWORD_TEST": "**********"' in caplog.text - assert 'secret_value' not in caplog.text + assert "secret_value" not in caplog.text -def test_log_all_should_return_only_the_custom_regex_data_masked_if_MASKED_SENSITIVE_SETTINGS_REGEX_LIST_configured(caplog): +def test_log_all_should_return_only_the_custom_regex_data_masked_if_MASKED_SENSITIVE_SETTINGS_REGEX_LIST_configured( + caplog, +): settings = { "SETTINGS_LOGGING_ENABLED": True, "MASKED_SENSITIVE_SETTINGS_REGEX_LIST": ["apppppppikey"], - "APIKEY": 'apikey_value1', - "apppppppikey": 'some_random_value', + "APIKEY": "apikey_value1", + "apppppppikey": "some_random_value", } spider = MockSpider(settings) @@ -192,6 +196,6 @@ def test_log_all_should_return_only_the_custom_regex_data_masked_if_MASKED_SENSI with caplog.at_level(logging.INFO): logger.spider_closed(spider) - assert 'apikey_value1' in caplog.text + assert "apikey_value1" in caplog.text assert '"apppppppikey": "**********"' in caplog.text - assert 'some_random_value' not in caplog.text \ No newline at end of file + assert "some_random_value" not in caplog.text From 73ab994026c7b0728264329b8327777891ff4fd3 Mon Sep 17 00:00:00 2001 From: Victor Ruiz Date: Tue, 20 Aug 2024 10:39:17 +0200 Subject: [PATCH 2/3] Add commit to git blame ignore revs --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 34afbf5..b5c6f6b 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1 +1,2 @@ d2d60666e425eeda24b7735702978b7dbef509f9 +eabf7df0eba1a7edf25712c3272d171d5aeb11ea From a1a8a84f0fa0b86cbe9596670dbd4557a4267ffa Mon Sep 17 00:00:00 2001 From: Victor Ruiz Date: Tue, 20 Aug 2024 10:51:17 +0200 Subject: [PATCH 3/3] Move global flag to start of the regexp --- src/scrapy_settings_log/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scrapy_settings_log/__init__.py b/src/scrapy_settings_log/__init__.py index 07ba54a..4aa139a 100644 --- a/src/scrapy_settings_log/__init__.py +++ b/src/scrapy_settings_log/__init__.py @@ -10,9 +10,9 @@ logger = logging.getLogger(__name__) DEFAULT_REGEXES = [ - r".*(?i)(api[\W_]*key).*", # apikey and variations e.g: shub_apikey or SC_APIKEY - r".*(?i)(AWS[\W_]*(SECRET[\W_]*)?(ACCESS)?[\W_]*(KEY|ACCESS[\W_]*KEY))", # AWS_SECRET_ACCESS_KEY and variations - r".*(?i)([\W_]*password[\W_]*).*", # password word + r"(?i).*(api[\W_]*key).*", # apikey and variations e.g: shub_apikey or SC_APIKEY + r"(?i).*(AWS[\W_]*(SECRET[\W_]*)?(ACCESS)?[\W_]*(KEY|ACCESS[\W_]*KEY))", # AWS_SECRET_ACCESS_KEY and variations + r"(?i).*([\W_]*password[\W_]*).*", # password word ]