From 3af0a1b6fb3025e4d257c1e5a3be587382c50af6 Mon Sep 17 00:00:00 2001 From: Karan Kohli Date: Mon, 29 Aug 2022 02:48:46 +0530 Subject: [PATCH 1/5] scan line --- detect_secrets/core/scan.py | 31 +++++++++++++++++++++++ detect_secrets/core/secrets_collection.py | 4 +++ detect_secrets/pre_commit_hook.py | 4 +++ test.py | 10 ++++++++ 4 files changed, 49 insertions(+) create mode 100644 test.py diff --git a/detect_secrets/core/scan.py b/detect_secrets/core/scan.py index 03bda2179..e1c5daf70 100644 --- a/detect_secrets/core/scan.py +++ b/detect_secrets/core/scan.py @@ -174,6 +174,37 @@ def scan_diff(diff: str) -> Generator[PotentialSecret, None, None]: yield from _process_line_based_plugins(lines, filename=filename) +def scan_line(filename: str, line: str) -> Generator[PotentialSecret, None, None]: + """ + :raises: ImportError + """ + if not get_plugins(): # pragma: no cover + log.error('No plugins to scan with!') + return + context = get_code_snippet(lines=[line], line_number=1) + yield from ( + secret + for plugin in get_plugins() + for secret in _scan_line( + plugin=plugin, + filename=filename, + line=line, + line_number=1, + context=context, + ) + if not _is_filtered_out( + required_filter_parameters=['context'], + filename=secret.filename, + secret=secret.secret_value, + plugin=plugin, + line=line, + context=context, + ) + ) + + # yield from _process_line_based_plugins(line, filename=filename) + + def scan_for_allowlisted_secrets_in_file(filename: str) -> Generator[PotentialSecret, None, None]: """ Developers are able to add individual lines to the allowlist using diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index d3fc4dd56..5859b986f 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -88,6 +88,10 @@ def scan_diff(self, diff: str) -> None: 'installing that package, and try again.', ) + def scan_line(self, filename: str, line: str) -> None: + for secret in scan.scan_line(filename, line): + self[secret.filename].add(secret) + def merge(self, old_results: 'SecretsCollection') -> None: """ We operate under an assumption that the latest results are always more accurate, diff --git a/detect_secrets/pre_commit_hook.py b/detect_secrets/pre_commit_hook.py index fb75832e3..acf98477e 100644 --- a/detect_secrets/pre_commit_hook.py +++ b/detect_secrets/pre_commit_hook.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import json import os @@ -18,6 +19,7 @@ def main(argv: Optional[List[str]] = None) -> int: try: args = parse_args(argv) + print(args) except ValueError: return 1 @@ -26,7 +28,9 @@ def main(argv: Optional[List[str]] = None) -> int: # Find all secrets in files to be committed secrets = SecretsCollection() + args.file for filename in args.filenames: + print(filename) secrets.scan_file(filename) new_secrets = secrets diff --git a/test.py b/test.py new file mode 100644 index 000000000..7f320718b --- /dev/null +++ b/test.py @@ -0,0 +1,10 @@ +from detect_secrets import SecretsCollection +from detect_secrets.settings import default_settings + +secrets = SecretsCollection() +with default_settings(): + secrets.scan_line('test.txt','XMY_KEY="AKIAY5XAIXXR7ALTOOHH"MY_SECRET="3L091LfFvV/dcoDCy5YO7sWz56bitLQjlVVY2WxmY"') + + +import json +print(json.dumps(secrets.json(), indent=2)) \ No newline at end of file From c1e8a8b9c96c3470d2eb5a719855348e7e99f874 Mon Sep 17 00:00:00 2001 From: Karan Kohli Date: Mon, 29 Aug 2022 02:58:28 +0530 Subject: [PATCH 2/5] Delete test.py --- test.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 7f320718b..000000000 --- a/test.py +++ /dev/null @@ -1,10 +0,0 @@ -from detect_secrets import SecretsCollection -from detect_secrets.settings import default_settings - -secrets = SecretsCollection() -with default_settings(): - secrets.scan_line('test.txt','XMY_KEY="AKIAY5XAIXXR7ALTOOHH"MY_SECRET="3L091LfFvV/dcoDCy5YO7sWz56bitLQjlVVY2WxmY"') - - -import json -print(json.dumps(secrets.json(), indent=2)) \ No newline at end of file From 71ec67c2bde54c245cdf5c3d9d9f040b4c3d6049 Mon Sep 17 00:00:00 2001 From: Karan Kohli Date: Mon, 29 Aug 2022 03:08:06 +0530 Subject: [PATCH 3/5] fixed --- detect_secrets/core/scan.py | 31 ----------------------- detect_secrets/core/secrets_collection.py | 4 +-- test.py | 2 +- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/detect_secrets/core/scan.py b/detect_secrets/core/scan.py index e1c5daf70..03bda2179 100644 --- a/detect_secrets/core/scan.py +++ b/detect_secrets/core/scan.py @@ -174,37 +174,6 @@ def scan_diff(diff: str) -> Generator[PotentialSecret, None, None]: yield from _process_line_based_plugins(lines, filename=filename) -def scan_line(filename: str, line: str) -> Generator[PotentialSecret, None, None]: - """ - :raises: ImportError - """ - if not get_plugins(): # pragma: no cover - log.error('No plugins to scan with!') - return - context = get_code_snippet(lines=[line], line_number=1) - yield from ( - secret - for plugin in get_plugins() - for secret in _scan_line( - plugin=plugin, - filename=filename, - line=line, - line_number=1, - context=context, - ) - if not _is_filtered_out( - required_filter_parameters=['context'], - filename=secret.filename, - secret=secret.secret_value, - plugin=plugin, - line=line, - context=context, - ) - ) - - # yield from _process_line_based_plugins(line, filename=filename) - - def scan_for_allowlisted_secrets_in_file(filename: str) -> Generator[PotentialSecret, None, None]: """ Developers are able to add individual lines to the allowlist using diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index 5859b986f..e48c790da 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -88,8 +88,8 @@ def scan_diff(self, diff: str) -> None: 'installing that package, and try again.', ) - def scan_line(self, filename: str, line: str) -> None: - for secret in scan.scan_line(filename, line): + def scan_line(self, line: str) -> None: + for secret in scan.scan_line(line): self[secret.filename].add(secret) def merge(self, old_results: 'SecretsCollection') -> None: diff --git a/test.py b/test.py index 7f320718b..942877301 100644 --- a/test.py +++ b/test.py @@ -3,7 +3,7 @@ secrets = SecretsCollection() with default_settings(): - secrets.scan_line('test.txt','XMY_KEY="AKIAY5XAIXXR7ALTOOHH"MY_SECRET="3L091LfFvV/dcoDCy5YO7sWz56bitLQjlVVY2WxmY"') + secrets.scan_line('XMY_KEY="AKIAY5XAIXXR7ALTOOHH"MY_SECRET="3L091LfFvV/dcoDCy5YO7sWz56bitLQjlVVY2WxmY"') import json From 8bd8f97a06dfa5b3768ea3832d10bf3cd71b5a43 Mon Sep 17 00:00:00 2001 From: Karan Kohli Date: Mon, 29 Aug 2022 23:55:35 +0530 Subject: [PATCH 4/5] redacted secret --- detect_secrets/core/potential_secret.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/detect_secrets/core/potential_secret.py b/detect_secrets/core/potential_secret.py index eba034c79..df5d3af07 100644 --- a/detect_secrets/core/potential_secret.py +++ b/detect_secrets/core/potential_secret.py @@ -1,4 +1,5 @@ import hashlib +import random from typing import Any from typing import Dict from typing import Optional @@ -53,7 +54,7 @@ def __init__( self.fields_to_compare = ['filename', 'secret_hash', 'type'] def set_secret(self, secret: str) -> None: - self.secret_hash: str = self.hash_secret(secret) + self.secret_hash: str = self.redact_secret(secret) # Note: Originally, we never wanted to keep the secret value in memory, # after finding it in the codebase. However, to support verifiable @@ -70,6 +71,13 @@ def hash_secret(secret: str) -> str: """This offers a way to coherently test this class, without mocking self.secret_hash.""" return hashlib.sha1(secret.encode('utf-8')).hexdigest() + @staticmethod + def redact_secret(secret: str) -> str: + temp = [k for k in secret] + for j in random.sample(range(len(secret) - 1),len(secret)//2): + temp[j] = '*' + return ''.join(temp) + @classmethod def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'PotentialSecret': """Custom JSON decoder""" From 51a9da701e728cdeeb08b2abc4101234ae507f7d Mon Sep 17 00:00:00 2001 From: Karan Kohli Date: Thu, 1 Sep 2022 04:18:55 +0530 Subject: [PATCH 5/5] change: redact string with hash string --- detect_secrets/core/potential_secret.py | 14 ++++++++++---- detect_secrets/plugins/base.py | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/detect_secrets/core/potential_secret.py b/detect_secrets/core/potential_secret.py index df5d3af07..74b52cb01 100644 --- a/detect_secrets/core/potential_secret.py +++ b/detect_secrets/core/potential_secret.py @@ -54,7 +54,9 @@ def __init__( self.fields_to_compare = ['filename', 'secret_hash', 'type'] def set_secret(self, secret: str) -> None: - self.secret_hash: str = self.redact_secret(secret) + if(len(secret) > 4): + self.secret_hash: str = self.hash_secret(secret) + self.secret_redacted: str = self.redact_secret(secret) # Note: Originally, we never wanted to keep the secret value in memory, # after finding it in the codebase. However, to support verifiable @@ -73,10 +75,12 @@ def hash_secret(secret: str) -> str: @staticmethod def redact_secret(secret: str) -> str: - temp = [k for k in secret] - for j in random.sample(range(len(secret) - 1),len(secret)//2): + temp = secret[2:-2] + temp = [k for k in temp] + for j in random.sample(range(len(temp) - 1),len(temp)//2): temp[j] = '*' - return ''.join(temp) + temp=''.join(temp) + return secret[0] + secret[1] + temp + secret[-2] + secret[-1] @classmethod def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'PotentialSecret': @@ -99,6 +103,7 @@ def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'Poten output = cls(**kwargs) output.secret_value = None output.secret_hash = str(data['hashed_secret']) + output.secret_redacted = str(data['redacted_secret']) return output @@ -108,6 +113,7 @@ def json(self) -> Dict[str, Union[str, int, bool]]: 'type': self.type, 'filename': self.filename, 'hashed_secret': self.secret_hash, + 'redacted_secret': self.secret_redacted, 'is_verified': self.is_verified, } diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index 01d0f6f14..23f492244 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -57,6 +57,8 @@ def analyze_line( is_verified: bool = False # If the filter is disabled it means --no-verify flag was passed # We won't run verification in that case + if(len(match) < 5): + return if ( 'detect_secrets.filters.common.is_ignored_due_to_verification_policies' in get_settings().filters