Skip to content

Commit

Permalink
Merge pull request #9 from scrapy-plugins/fix/linting
Browse files Browse the repository at this point in the history 
Fix/linting
  • Loading branch information
@VMRuiz
VMRuiz authored Aug 20, 2024
2 parents 87b05e7 + a1a8a84 commit 11230b6
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 25 deletions.
1 change: 1 addition & 0 deletions .git-blame-ignore-revs
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
d2d60666e425eeda24b7735702978b7dbef509f9
eabf7df0eba1a7edf25712c3272d171d5aeb11ea
16 changes: 11 additions & 5 deletions src/scrapy_settings_log/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
logger = logging.getLogger(__name__)

DEFAULT_REGEXES = [
".*(?i)(api[\W_]*key).*", # apikey and variations e.g: shub_apikey or SC_APIKEY
".*(?i)(AWS[\W_]*(SECRET[\W_]*)?(ACCESS)?[\W_]*(KEY|ACCESS[\W_]*KEY))", # AWS_SECRET_ACCESS_KEY and variations
".*(?i)([\W_]*password[\W_]*).*" # password word
r"(?i).*(api[\W_]*key).*", # apikey and variations e.g: shub_apikey or SC_APIKEY
r"(?i).*(AWS[\W_]*(SECRET[\W_]*)?(ACCESS)?[\W_]*(KEY|ACCESS[\W_]*KEY))", # AWS_SECRET_ACCESS_KEY and variations
r"(?i).*([\W_]*password[\W_]*).*", # password word
]


Expand Down Expand Up @@ -58,9 +58,15 @@ def spider_closed(self, spider):
if regex is not None:
settings = {k: v for k, v in settings.items() if re.search(regex, k)}
if spider.settings.getbool("MASKED_SENSITIVE_SETTINGS_ENABLED", True):
regex_list = spider.settings.getlist("MASKED_SENSITIVE_SETTINGS_REGEX_LIST", DEFAULT_REGEXES)
regex_list = spider.settings.getlist(
"MASKED_SENSITIVE_SETTINGS_REGEX_LIST", DEFAULT_REGEXES
)
for reg in regex_list:
updated_settings = {k: '**********' if v else v for k, v in settings.items() if re.match(reg, k)}
updated_settings = {
k: "**********" if v else v
for k, v in settings.items()
if re.match(reg, k)
}
settings = {**settings, **updated_settings}

self.output_settings(settings, spider)
Expand Down
44 changes: 24 additions & 20 deletions tests/test_code.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,9 @@ class CustomClass:
def test_log_all_should_not_return_apikey_value_by_default(caplog):
settings = {
"SETTINGS_LOGGING_ENABLED": True,
"SHUB_APIKEY": 'apikey_value1',
"shub_apikey": 'apikey_value2',
"api_key": 'apikey_value3',
"SHUB_APIKEY": "apikey_value1",
"shub_apikey": "apikey_value2",
"api_key": "apikey_value3",
}

spider = MockSpider(settings)
Expand All @@ -121,13 +121,15 @@ def test_log_all_should_not_return_apikey_value_by_default(caplog):
assert '"SHUB_APIKEY": "**********"' in caplog.text
assert '"shub_apikey": "**********"' in caplog.text
assert '"api_key": "**********"' in caplog.text
assert 'apikey_value' not in caplog.text
assert "apikey_value" not in caplog.text


def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED_is_false(caplog):
def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED_is_false(
caplog,
):
settings = {
"SETTINGS_LOGGING_ENABLED": True,
"APIKEY": 'apikey_value',
"APIKEY": "apikey_value",
"MASKED_SENSITIVE_SETTINGS_ENABLED": False,
}

Expand All @@ -142,11 +144,11 @@ def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED
def test_log_all_should_not_return_aws_secret_key_value_by_default(caplog):
settings = {
"SETTINGS_LOGGING_ENABLED": True,
"AWS_SECRET_ACCESS_KEY": 'secret_value1',
"aws_secret_access_key": 'secret_value2',
"aws_access_key": 'secret_value2',
"AWS_SECRET_KEY": 'secret_value2',
"aws_secret_key": 'secret_value2',
"AWS_SECRET_ACCESS_KEY": "secret_value1",
"aws_secret_access_key": "secret_value2",
"aws_access_key": "secret_value2",
"AWS_SECRET_KEY": "secret_value2",
"aws_secret_key": "secret_value2",
}

spider = MockSpider(settings)
Expand All @@ -159,14 +161,14 @@ def test_log_all_should_not_return_aws_secret_key_value_by_default(caplog):
assert '"aws_access_key": "**********"' in caplog.text
assert '"AWS_SECRET_KEY": "**********"' in caplog.text
assert '"aws_secret_key": "**********"' in caplog.text
assert 'secret_value' not in caplog.text
assert "secret_value" not in caplog.text


def test_log_all_should_not_return_password_value_by_default(caplog):
settings = {
"SETTINGS_LOGGING_ENABLED": True,
"test_password": 'secret_value1',
"PASSWORD_TEST": 'secret_value2',
"test_password": "secret_value1",
"PASSWORD_TEST": "secret_value2",
}

spider = MockSpider(settings)
Expand All @@ -176,22 +178,24 @@ def test_log_all_should_not_return_password_value_by_default(caplog):

assert '"test_password": "**********"' in caplog.text
assert '"PASSWORD_TEST": "**********"' in caplog.text
assert 'secret_value' not in caplog.text
assert "secret_value" not in caplog.text


def test_log_all_should_return_only_the_custom_regex_data_masked_if_MASKED_SENSITIVE_SETTINGS_REGEX_LIST_configured(caplog):
def test_log_all_should_return_only_the_custom_regex_data_masked_if_MASKED_SENSITIVE_SETTINGS_REGEX_LIST_configured(
caplog,
):
settings = {
"SETTINGS_LOGGING_ENABLED": True,
"MASKED_SENSITIVE_SETTINGS_REGEX_LIST": ["apppppppikey"],
"APIKEY": 'apikey_value1',
"apppppppikey": 'some_random_value',
"APIKEY": "apikey_value1",
"apppppppikey": "some_random_value",
}

spider = MockSpider(settings)
logger = SpiderSettingsLogging()
with caplog.at_level(logging.INFO):
logger.spider_closed(spider)

assert 'apikey_value1' in caplog.text
assert "apikey_value1" in caplog.text
assert '"apppppppikey": "**********"' in caplog.text
assert 'some_random_value' not in caplog.text
assert "some_random_value" not in caplog.text

0 comments on commit 11230b6

Please sign in to comment.