diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 000000000..4aab873b8 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,20 @@ +* **I'm submitting a ...** + - [ ] bug report + - [ ] feature request + +* **What is the current behavior?** + +* **If the current behavior is a bug, please provide the steps to reproduce and if possible a minimal demo of the problem** + +* **What is the expected behavior?** + +* **What is the motivation / use case for changing the behavior?** + +* **Please tell us about your environment:** + - detect-secrets Version: + - Python Version: + - OS Version: + - File type (if applicable): + +* **Other information** + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..43d13cdd6 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ +* **Please check if the PR fulfills these requirements** +- [ ] Tests for the changes have been added + +- [ ] Docs have been added / updated + +- [ ] All CI checks are green + +* **What kind of change does this PR introduce?** + + +* **What is the current behavior?** + + +* **What is the new behavior (if this is a feature change)?** + +* **Does this PR introduce a breaking change?** + + +* **Other information**: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6cb99bfc..bfe15f4b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,8 @@ on: branches: [ master ] pull_request: workflow_dispatch: + schedule: + - cron: '0 0 1 * *' jobs: main: @@ -15,7 +17,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.6', '3.7', '3.8', '3.9'] + python: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index f96741176..13b128ace 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python: ['3.6', '3.7', '3.8', '3.9'] + python: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/.secrets.baseline b/.secrets.baseline index 6201d1f0c..707710ecc 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,5 +1,5 @@ { - "version": "1.3.0", + "version": "1.4.0", "plugins_used": [ { "name": "ArtifactoryDetector" diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ddd3efa..2f2f363a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,43 @@ If you love `detect-secrets`, please star our project on GitHub to show your sup ### Unreleased --> +### v1.4.0 +##### October 4th, 2022 + +#### :newspaper: News +- We're dropping support for Python 3.6 starting v1.5.0! Python 3.6 reached EOL on December 23, 2021 and, therefore, is currently unsupported. We hope this announcement gives you plenty of time to upgrade your project, if needed. + +#### :mega: Release Highlights +- Improved filtering by excluding secrets that have already been detected by a regex-based detector ([#612]) +#### :tada: New Features +- Added a detector for Discord bot tokens ([#614]) + +#### :sparkles: Usability +- Improved the audit report to make it easier to parse programmatically ([#619]) +#### :telescope: Accuracy +- Improve ArtifactoryDetector plugin to reduce false positives ([#499]) + +#### :bug: Bugfixes +- Fixed the verify flow in audit report by adding the code snippet of the verified secret ([#620]) +- Fixed deploy process to be environment configuration independent ([#625]) + +#### :snake: Miscellaneous +- Added support for .NET packages.lock.json files in the heuristic filter ([#593]) +- Multiple dependency updates + +[#499]: https://github.com/Yelp/detect-secrets/pull/499 +[#556]: https://github.com/Yelp/detect-secrets/pull/556 +[#589]: https://github.com/Yelp/detect-secrets/pull/589 +[#593]: https://github.com/Yelp/detect-secrets/pull/593 +[#598]: https://github.com/Yelp/detect-secrets/pull/598 +[#612]: https://github.com/Yelp/detect-secrets/pull/612 +[#614]: https://github.com/Yelp/detect-secrets/pull/614 +[#615]: https://github.com/Yelp/detect-secrets/pull/615 +[#616]: https://github.com/Yelp/detect-secrets/pull/616 +[#619]: https://github.com/Yelp/detect-secrets/pull/619 +[#620]: https://github.com/Yelp/detect-secrets/pull/620 +[#625]: https://github.com/Yelp/detect-secrets/pull/625 + ### v1.3.0 ##### July 22nd, 2022 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af1fc3601..dbd7f723b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,10 +70,10 @@ sys 0m2.486s ### Running the Entire Test Suite -You can run the test suite in the interpreter of your choice (in this example, `py36`) by doing: +You can run the test suite in the interpreter of your choice (in this example, `py37`) by doing: ```bash -tox -e py36 +tox -e py37 ``` This will also run the code through our series of coverage tests, `mypy` rules and other linting diff --git a/README.md b/README.md index ac101893e..03e7e36fb 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,8 @@ AWSKeyDetector AzureStorageKeyDetector BasicAuthDetector CloudantDetector +DiscordBotTokenDetector +GitHubTokenDetector Base64HighEntropyString HexHighEntropyString IbmCloudIamDetector @@ -105,8 +107,10 @@ KeywordDetector MailchimpDetector NpmDetector PrivateKeyDetector +SendGridDetector SlackDetector SoftlayerDetector +SquareOAuthDetector StripeDetector TwilioKeyDetector ``` @@ -388,7 +392,7 @@ We recommend setting this up as a pre-commit hook. One way to do this is by usin # .pre-commit-config.yaml repos: - repo: https://github.com/Yelp/detect-secrets - rev: v1.3.0 + rev: v1.4.0 hooks: - id: detect-secrets args: ['--baseline', '.secrets.baseline'] diff --git a/detect_secrets/__version__.py b/detect_secrets/__version__.py index 9e2335938..b2e817771 100644 --- a/detect_secrets/__version__.py +++ b/detect_secrets/__version__.py @@ -1 +1 @@ -VERSION = '1.3.0' +VERSION = '1.4.0' diff --git a/detect_secrets/audit/analytics.py b/detect_secrets/audit/analytics.py index 29facdbbb..c46aa491d 100644 --- a/detect_secrets/audit/analytics.py +++ b/detect_secrets/audit/analytics.py @@ -58,7 +58,14 @@ def _get_plugin_counter(self, secret_type: str) -> 'StatisticsCounter': return cast(StatisticsCounter, self.data[secret_type]['stats']) def __str__(self) -> str: - raise NotImplementedError + output = '' + + for secret_type, framework in self.data.items(): + output += f'Plugin: {get_mapping_from_secret_type_to_class()[secret_type].__name__}\n' + for value in framework.values(): + output += f'Statistics: {value}\n\n' + + return output def json(self) -> Dict[str, Any]: output = {} @@ -77,19 +84,36 @@ def __init__(self) -> None: self.incorrect: int = 0 self.unknown: int = 0 - def __repr__(self) -> str: + def __str__(self) -> str: return ( - f'{self.__class__.__name__}(correct={self.correct}, ' - 'incorrect={self.incorrect}, unknown={self.unknown},)' + f'True Positives: {self.correct}, False Positives: {self.incorrect}, ' + f'Unknown: {self.unknown}, Precision: {self.calculate_precision()}, ' + f'Recall: {self.calculate_recall()}' ) def json(self) -> Dict[str, Any]: + return { + 'raw': { + 'true-positives': self.correct, + 'false-positives': self.incorrect, + 'unknown': self.unknown, + }, + 'score': { + 'precision': self.calculate_precision(), + 'recall': self.calculate_recall(), + }, + } + + def calculate_precision(self) -> float: precision = ( round(float(self.correct) / (self.correct + self.incorrect), 4) if (self.correct and self.incorrect) else 0.0 ) + return precision + + def calculate_recall(self) -> float: # NOTE(2020-11-08|domanchi): This isn't the formal definition of `recall`, however, # this is the definition that we're going to attribute to it. # @@ -124,14 +148,4 @@ def json(self) -> Dict[str, Any]: else 0.0 ) - return { - 'raw': { - 'true-positives': self.correct, - 'false-positives': self.incorrect, - 'unknown': self.unknown, - }, - 'score': { - 'precision': precision, - 'recall': recall, - }, - } + return recall diff --git a/detect_secrets/audit/common.py b/detect_secrets/audit/common.py index 7359fc419..ca0bb2b76 100644 --- a/detect_secrets/audit/common.py +++ b/detect_secrets/audit/common.py @@ -19,6 +19,7 @@ from ..transformers import get_transformed_file from ..types import NamedIO from ..util.inject import call_function_with_arguments +from detect_secrets.util.code_snippet import get_code_snippet def get_baseline_from_file(filename: str) -> SecretsCollection: @@ -91,6 +92,7 @@ def get_raw_secrets_from_file( line_numbers = list(range(len(lines_to_scan))) for line_number, line in zip(line_numbers, lines_to_scan): + context = get_code_snippet(lines=line_getter.lines, line_number=line_number + 1) identified_secrets = call_function_with_arguments( plugin.analyze_line, filename=secret.filename, @@ -100,6 +102,7 @@ def get_raw_secrets_from_file( # We enable eager search, because we *know* there's a secret here -- the baseline # flagged it after all. enable_eager_search=bool(secret.line_number), + context=context, ) for identified_secret in (identified_secrets or []): diff --git a/detect_secrets/audit/report.py b/detect_secrets/audit/report.py index ce1536b04..7b16fb5d7 100644 --- a/detect_secrets/audit/report.py +++ b/detect_secrets/audit/report.py @@ -28,7 +28,7 @@ def generate_report( baseline_file: str, class_to_print: SecretClassToPrint = None, line_getter_factory: Callable[[str], 'LineGetter'] = open_file, -) -> List[Dict[str, Any]]: +) -> Dict[str, List[Dict[str, Any]]]: secrets: Dict[Tuple[str, str], Any] = {} for filename, secret in get_baseline_from_file(baseline_file): @@ -63,8 +63,9 @@ def generate_report( ], 'category': verified_result.name, } - - return list(secrets.values()) + return { + 'results': list(secrets.values()), + } def get_prioritized_verified_result( diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index f71a821c1..16c9f45f3 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -128,7 +128,7 @@ def upgrade(baseline: Dict[str, Any]) -> Dict[str, Any]: new_baseline = {**baseline} for module in modules: - module.upgrade(new_baseline) # type: ignore + module.upgrade(new_baseline) new_baseline['version'] = VERSION return new_baseline diff --git a/detect_secrets/core/plugins/initialize.py b/detect_secrets/core/plugins/initialize.py index 36bbea5be..6d8eeaeb9 100644 --- a/detect_secrets/core/plugins/initialize.py +++ b/detect_secrets/core/plugins/initialize.py @@ -21,7 +21,7 @@ def from_secret_type(secret_type: str) -> Plugin: raise TypeError try: - return plugin_type(**_get_config(plugin_type.__name__)) # type: ignore + return plugin_type(**_get_config(plugin_type.__name__)) except TypeError: log.error('Unable to initialize plugin!') raise @@ -44,7 +44,7 @@ def from_plugin_classname(classname: str) -> Plugin: raise TypeError try: - return plugin_type(**_get_config(classname)) # type: ignore + return plugin_type(**_get_config(classname)) except TypeError: log.error('Unable to initialize plugin!') raise diff --git a/detect_secrets/core/scan.py b/detect_secrets/core/scan.py index 03bda2179..f84d53c3c 100644 --- a/detect_secrets/core/scan.py +++ b/detect_secrets/core/scan.py @@ -101,7 +101,7 @@ def get_files_to_scan( if ( valid_paths is True - or relative_path in cast(Set[str], valid_paths) + or relative_path in valid_paths ): yield relative_path diff --git a/detect_secrets/core/usage/audit.py b/detect_secrets/core/usage/audit.py index 5bb5ec66c..2271410c6 100644 --- a/detect_secrets/core/usage/audit.py +++ b/detect_secrets/core/usage/audit.py @@ -1,4 +1,5 @@ import argparse +from typing import cast def add_audit_action(parent: argparse._SubParsersAction) -> argparse.ArgumentParser: @@ -23,7 +24,7 @@ def add_audit_action(parent: argparse._SubParsersAction) -> argparse.ArgumentPar _add_mode_parser(parser) _add_report_module(parser) _add_statistics_module(parser) - return parser + return cast(argparse.ArgumentParser, parser) def _add_mode_parser(parser: argparse.ArgumentParser) -> None: diff --git a/detect_secrets/core/usage/scan.py b/detect_secrets/core/usage/scan.py index 6f0095578..f16ba3300 100644 --- a/detect_secrets/core/usage/scan.py +++ b/detect_secrets/core/usage/scan.py @@ -20,7 +20,7 @@ def add_scan_action(parent: argparse._SubParsersAction) -> argparse.ArgumentPars _add_pragma_scanning(parser) _add_initialize_baseline_options(parser) - return parser + return cast(argparse.ArgumentParser, parser) def _add_adhoc_scanning(parser: argparse.ArgumentParser) -> None: diff --git a/detect_secrets/filters/gibberish/__init__.py b/detect_secrets/filters/gibberish/__init__.py index 977a76cfc..651cf3ff2 100644 --- a/detect_secrets/filters/gibberish/__init__.py +++ b/detect_secrets/filters/gibberish/__init__.py @@ -32,7 +32,7 @@ def initialize(model_path: Optional[str] = None, limit: float = 3.7) -> None: """ path = model_path if not path: - path = os.path.join(__path__[0], 'rfc.model') # type: ignore # mypy issue #1422 + path = os.path.join(__path__[0], 'rfc.model') model = get_model() diff --git a/detect_secrets/filters/heuristic.py b/detect_secrets/filters/heuristic.py index a7b661668..7fb078181 100644 --- a/detect_secrets/filters/heuristic.py +++ b/detect_secrets/filters/heuristic.py @@ -2,8 +2,12 @@ import re import string from functools import lru_cache +from typing import Optional from typing import Pattern +from detect_secrets.plugins.base import BasePlugin +from detect_secrets.plugins.base import RegexBasedDetector + def is_sequential_string(secret: str) -> bool: sequences = ( @@ -57,13 +61,14 @@ def _get_uuid_regex() -> Pattern: ) -def is_likely_id_string(secret: str, line: str) -> bool: +def is_likely_id_string(secret: str, line: str, plugin: Optional[BasePlugin] = None) -> bool: try: index = line.index(secret) except ValueError: return False - return bool(_get_id_detector_regex().search(line, pos=0, endpos=index)) + return (not plugin or not isinstance(plugin, RegexBasedDetector)) \ + and bool(_get_id_detector_regex().search(line, pos=0, endpos=index)) @lru_cache(maxsize=1) @@ -159,7 +164,7 @@ def is_prefixed_with_dollar_sign(secret: str) -> bool: # false negatives than `is_templated_secret` (e.g. secrets that actually start with a $). # This is best used with files that actually use this as a means of referencing variables. # TODO: More intelligent filetype handling? - return secret[0] == '$' + return bool(secret) and secret[0] == '$' def is_indirect_reference(line: str) -> bool: @@ -208,6 +213,7 @@ def is_lock_file(filename: str) -> bool: 'Pipfile.lock', 'poetry.lock', 'Cargo.lock', + 'packages.lock.json', } diff --git a/detect_secrets/plugins/artifactory.py b/detect_secrets/plugins/artifactory.py index 0d0ee6a8d..d82f18d4a 100644 --- a/detect_secrets/plugins/artifactory.py +++ b/detect_secrets/plugins/artifactory.py @@ -9,7 +9,7 @@ class ArtifactoryDetector(RegexBasedDetector): denylist = [ # Artifactory tokens begin with AKC - re.compile(r'(?:\s|=|:|"|^)AKC[a-zA-Z0-9]{10,}'), # API token + re.compile(r'(?:\s|=|:|"|^)AKC[a-zA-Z0-9]{10,}(?:\s|"|$)'), # API token # Artifactory encrypted passwords begin with AP[A-Z] - re.compile(r'(?:\s|=|:|"|^)AP[\dABCDEF][a-zA-Z0-9]{8,}'), # Password + re.compile(r'(?:\s|=|:|"|^)AP[\dABCDEF][a-zA-Z0-9]{8,}(?:\s|"|$)'), # Password ] diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index 835bc26b8..cfdcffcba 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -53,11 +53,11 @@ def analyze_line( ) -> Set[PotentialSecret]: """This examines a line and finds all possible secret values in it.""" output = set() - for match in self.analyze_string(line, **kwargs): # type: ignore + for match in self.analyze_string(line, **kwargs): is_verified: bool = False # If the filter is disabled it means --no-verify flag was passed # We won't run verification in that case - if( + if ( 'detect_secrets.filters.common.is_ignored_due_to_verification_policies' in get_settings().filters ): diff --git a/detect_secrets/plugins/discord.py b/detect_secrets/plugins/discord.py new file mode 100644 index 000000000..67664fc22 --- /dev/null +++ b/detect_secrets/plugins/discord.py @@ -0,0 +1,18 @@ +""" +This plugin searches for Discord Bot Token +""" +import re + +from .base import RegexBasedDetector + + +class DiscordBotTokenDetector(RegexBasedDetector): + """Scans for Discord Bot token.""" + secret_type = 'Discord Bot Token' + + denylist = [ + # Discord Bot Token ([M|N|O]XXXXXXXXXXXXXXXXXXXXXXX[XX].XXXXXX.XXXXXXXXXXXXXXXXXXXXXXXXXXX) + # Reference: https://discord.com/developers/docs/reference#authentication + # Also see: https://github.com/Yelp/detect-secrets/issues/627 + re.compile(r'[MNO][a-zA-Z\d_-]{23,25}\.[a-zA-Z\d_-]{6}\.[a-zA-Z\d_-]{27}'), + ] diff --git a/detect_secrets/plugins/ibm_cloud_iam.py b/detect_secrets/plugins/ibm_cloud_iam.py index fdb016254..6920849c6 100644 --- a/detect_secrets/plugins/ibm_cloud_iam.py +++ b/detect_secrets/plugins/ibm_cloud_iam.py @@ -1,4 +1,3 @@ -from typing import cast from typing import Union import requests @@ -35,8 +34,8 @@ def verify(self, secret: str) -> VerifiedResult: def verify_cloud_iam_api_key(apikey: Union[str, bytes]) -> requests.Response: # pragma: no cover - if type(apikey) == bytes: - apikey = cast(bytes, apikey).decode('UTF-8') + if type(apikey) is bytes: + apikey = apikey.decode('UTF-8') headers = { 'Content-Type': 'application/x-www-form-urlencoded', diff --git a/detect_secrets/plugins/keyword.py b/detect_secrets/plugins/keyword.py index d5ee43014..e6c1366d7 100644 --- a/detect_secrets/plugins/keyword.py +++ b/detect_secrets/plugins/keyword.py @@ -98,7 +98,7 @@ FOLLOWED_BY_COLON_EQUAL_SIGNS_REGEX = re.compile( # e.g. my_password := "bar" or my_password := bar - r'{denylist}({closing})?{whitespace}:=?{whitespace}({quote}?)({secret})(\3)'.format( + r'{denylist}({closing})?{whitespace}:={whitespace}({quote}?)({secret})(\3)'.format( denylist=DENYLIST_REGEX, closing=CLOSING, quote=QUOTE, diff --git a/detect_secrets/transformers/yaml.py b/detect_secrets/transformers/yaml.py index c3cc4c1ac..a587703b6 100644 --- a/detect_secrets/transformers/yaml.py +++ b/detect_secrets/transformers/yaml.py @@ -37,8 +37,17 @@ def parse_file(self, file: NamedIO) -> List[str]: except yaml.YAMLError: raise ParsingError + seen = set() + lines: List[str] = [] for item in items: + # Filter out previous lines seen before. This removes duplicates when it comes + # to anchor & and alias * tags. + if item in seen: + continue + else: + seen.add(item) + while len(lines) < item.line_number - 1: lines.append('') @@ -210,25 +219,28 @@ def _compose_node_shim( self, parent: Optional[yaml.nodes.Node], index: Optional[yaml.nodes.Node], - ) -> yaml.nodes.Node: + ) -> Optional[yaml.nodes.Node]: line = ( self.loader.marks[-1].line if self.is_inline_flow_mapping_key else self.loader.line ) - node = yaml.composer.Composer.compose_node(self.loader, parent, index) - node.__line__ = line + 1 + node = yaml.composer.Composer.compose_node(self.loader, parent, index) # type: ignore + if node is None: + return None + + node.__line__ = line + 1 # type: ignore if node.tag.endswith(':map'): # Reset the inline flow mapping key when the end of a mapping is reached # to avoid complications with empty mappings self.is_inline_flow_mapping_key = False - return _tag_dict_values(node) + return _tag_dict_values(cast(yaml.nodes.MappingNode, node)) # TODO: Not sure if need to do :seq - return cast(yaml.nodes.Node, node) + return node def _parse_flow_mapping_key_shim( self, diff --git a/detect_secrets/types.py b/detect_secrets/types.py index 92c155832..71c280f43 100644 --- a/detect_secrets/types.py +++ b/detect_secrets/types.py @@ -9,25 +9,6 @@ from .exceptions import SecretNotFoundOnSpecifiedLineError from .util.code_snippet import CodeSnippet -try: - from typing import NoReturn # noqa: F811 -except ImportError: # pragma: no cover - # NOTE: NoReturn was introduced in Python3.6.2. However, we need to support Python3.6.0. - # This section of code is inline imported from `typing-extensions`, so that we don't need - # to introduce an additional package for such an edge case. - from typing import _FinalTypingBase # type: ignore - - class _NoReturn(_FinalTypingBase): - __slots__ = () - - def __instancecheck__(self, obj: Any) -> None: - raise TypeError('NoReturn cannot be used with isinstance().') - - def __subclasscheck__(self, cls: Any) -> None: - raise TypeError('NoReturn cannot be used with issubclass().') - - NoReturn = _NoReturn(_root=True) # type: ignore - class SelfAwareCallable: """ diff --git a/detect_secrets/util/importlib.py b/detect_secrets/util/importlib.py index 7e2a3987f..f5e90005c 100644 --- a/detect_secrets/util/importlib.py +++ b/detect_secrets/util/importlib.py @@ -108,7 +108,7 @@ def get_modules_from_package(root: ModuleType) -> Iterable[str]: return [ module for _, module, is_package in pkgutil.walk_packages( - root.__path__, prefix=f'{root.__name__}.', # type: ignore # mypy issue #1422 + root.__path__, prefix=f'{root.__name__}.', ) if not is_package ] diff --git a/detect_secrets/util/inject.py b/detect_secrets/util/inject.py index 0c0981f46..3d104d2ad 100644 --- a/detect_secrets/util/inject.py +++ b/detect_secrets/util/inject.py @@ -27,7 +27,7 @@ def call_function_with_arguments( if inspect.ismethod(func) and not inspect.ismethod(function): # We also use get_injectable_variables (instead of hardcoding "self") to make sure that # this also handles cases where the developer doesn't name the first parameter `self`. - kwargs[get_injectable_variables(func)[0]] = func.__self__ # type: ignore + kwargs[get_injectable_variables(func)[0]] = func.__self__ variables_to_inject = set(kwargs.keys()) values = { @@ -49,7 +49,7 @@ def make_function_self_aware(func: Callable) -> SelfAwareCallable: # We can't add arbitrary attributes to methods, but we can to functions. Therefore, # we need to reference the underlying function itself. if inspect.ismethod(func): - klass = func.__self__.__class__ # type: ignore + klass = func.__self__.__class__ function = getattr(klass, func.__name__) function.injectable_variables = set(get_injectable_variables(func)) diff --git a/requirements-dev-minimal.txt b/requirements-dev-minimal.txt index e20833aaf..55f350114 100644 --- a/requirements-dev-minimal.txt +++ b/requirements-dev-minimal.txt @@ -1,6 +1,4 @@ -# coveragepy==5.0 fails with `Safety level may not be changed inside a transaction -# on python 3.6.0 (xenial) -coverage<5 +coverage flake8==3.5.0 gibberish-detector>=0.1.1 monotonic diff --git a/requirements-dev.txt b/requirements-dev.txt index 1294e7861..60a7a9824 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,42 +1,43 @@ -attrs==21.4.0 -backports.entry-points-selectable==1.1.1 -certifi==2021.10.8 -cfgv==3.2.0 -charset-normalizer==2.0.7 -coverage==4.5.4 -distlib==0.3.4 -filelock==3.0.12 -flake8==3.5.0 +attrs==23.2.0 +backports.entry-points-selectable==1.3.0 +certifi==2023.11.17 +cfgv==3.4.0 +charset-normalizer==3.3.2 +coverage==7.4.0 +distlib==0.3.8 +filelock==3.13.1 +flake8==6.1.0 gibberish-detector==0.1.1 -identify==2.3.0 -idna==3.3 -importlib-metadata==4.8.1 -iniconfig==1.1.1 -mccabe==0.6.1 +identify==2.5.33 +idna==3.6 +iniconfig==2.0.0 +mccabe==0.7.0 monotonic==1.6 -mypy==0.790 -mypy-extensions==0.4.3 -nodeenv==1.6.0 -packaging==21.3 -platformdirs==2.0.2 -pluggy==0.13.1 -pre-commit==2.17.0 +mypy==0.971 +mypy-extensions==1.0.0 +nodeenv==1.8.0 +packaging==23.2 +platformdirs==4.1.0 +pluggy==1.3.0 +pre-commit==3.5.0 py==1.11.0 -pyahocorasick==1.4.4 -pycodestyle==2.3.1 -pyflakes==1.6.0 -pyparsing==2.4.7 -pytest==6.2.2 -PyYAML==6.0 -requests==2.26.0 -responses==0.16.0 +pyahocorasick==2.0.0 +pycodestyle==2.11.1 +pyflakes==3.1.0 +pyparsing==3.1.1 +pytest==7.4.3 +PyYAML==6.0.1 +requests==2.31.0 +responses==0.24.1 six==1.16.0 toml==0.10.2 -tox==3.24.4 +tox==4.11.4 tox-pip-extensions==1.6.0 -typed-ast==1.4.3 -typing-extensions==3.10.0.2 -unidiff==0.7.3 -urllib3==1.26.9 -virtualenv==20.6.0 -zipp==3.6.0 +typed-ast==1.5.5 +types-PyYAML==6.0.12.12 +types-requests==2.31.0.20240106 +typing-extensions==4.9.0 +unidiff==0.7.5 +urllib3==2.1.0 +virtualenv==20.25.0 +zipp==3.17.0 diff --git a/scripts/bump-version b/scripts/bump-version index 95878c48a..9de611bc3 100755 --- a/scripts/bump-version +++ b/scripts/bump-version @@ -79,6 +79,9 @@ function installDependency() { # NOTE: We don't specify this in requirements-dev-minimal, since not all developers need # to bump the version. venv/bin/pip install bump2version + # Install local version of detect-secrets since when performing a commit, the pre-commit + # hook detect-secrets version (old) will be out of sync with the .secrets.baseline version (new) + venv/bin/pip install -e . } function setVersion() { diff --git a/setup.cfg b/setup.cfg index ae6c9d5ad..30a1b76ff 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.0 +current_version = 1.4.0 commit = True tag = True diff --git a/setup.py b/setup.py index 0ba463850..3613810f8 100644 --- a/setup.py +++ b/setup.py @@ -24,9 +24,8 @@ def get_version(): description='Tool for detecting secrets in the codebase', long_description=long_description, long_description_content_type='text/markdown', - license='Copyright Yelp, Inc. 2020', - author='Aaron Loo', - author_email='aaronloo@yelp.com', + author='Yelp, Inc.', + author_email='opensource@yelp.com', url='https://github.com/Yelp/detect-secrets', download_url='https://github.com/Yelp/detect-secrets/archive/{}.tar.gz'.format(VERSION), keywords=['secret-management', 'pre-commit', 'security', 'entropy-checks'], diff --git a/tests/audit/analytics_test.py b/tests/audit/analytics_test.py index 3938ce2fd..a5a099040 100644 --- a/tests/audit/analytics_test.py +++ b/tests/audit/analytics_test.py @@ -66,9 +66,14 @@ def test_no_divide_by_zero(secret): main(['audit', f.name, '--stats', '--json']) -@pytest.mark.skip(reason='TODO') -def test_basic_statistics_str(): - pass +def test_basic_statistics_str(printer): + with labelled_secrets() as filename: + main(['audit', filename, '--stats']) + + assert printer.message == ( + 'Plugin: BasicAuthDetector\nStatistics: True Positives: 1, ' + + 'False Positives: 2, Unknown: 1, Precision: 0.3333, Recall: 0.5\n\n\n' + ) @contextmanager diff --git a/tests/audit/report_test.py b/tests/audit/report_test.py index abdef414f..a83e3585d 100644 --- a/tests/audit/report_test.py +++ b/tests/audit/report_test.py @@ -10,6 +10,7 @@ from detect_secrets.constants import VerifiedResult from detect_secrets.core import baseline from detect_secrets.core.secrets_collection import SecretsCollection +from detect_secrets.plugins.aws import AWSKeyDetector from detect_secrets.plugins.basic_auth import BasicAuthDetector from detect_secrets.plugins.jwt import JwtTokenDetector from detect_secrets.settings import transient_settings @@ -20,115 +21,145 @@ first_secret = 'value1' second_secret = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ' # noqa: E501 random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(8)) +aws_secret = 'AKIAZZZZZZZZZZZZZZZZ' @pytest.mark.parametrize( 'class_to_print, expected_real, expected_false, expected_output', [ ( - None, 3, 1, [ - { - 'category': 'VERIFIED_TRUE', - 'lines': { - 1: 'url = {}'.format(url_format.format(first_secret)), - 3: 'link = {}'.format(url_format.format(first_secret)), + None, 4, 1, + { + 'results': [ + { + 'category': 'VERIFIED_TRUE', + 'lines': { + 1: 'url = {}'.format(url_format.format(first_secret)), + 3: 'link = {}'.format(url_format.format(first_secret)), + }, + 'secrets': first_secret, + 'types': [ + BasicAuthDetector.secret_type, + ], }, - 'secrets': first_secret, - 'types': [ - BasicAuthDetector.secret_type, - ], - }, - { - 'category': 'UNVERIFIED', - 'lines': { - 2: 'example = {}'.format(url_format.format(random_secret)), + { + 'category': 'UNVERIFIED', + 'lines': { + 2: 'example = {}'.format(url_format.format(random_secret)), + }, + 'secrets': random_secret, + 'types': [ + BasicAuthDetector.secret_type, + ], }, - 'secrets': random_secret, - 'types': [ - BasicAuthDetector.secret_type, - ], - }, - { - 'category': 'VERIFIED_TRUE', - 'lines': { - 1: 'url = {}'.format(url_format.format(second_secret)), + { + 'category': 'VERIFIED_TRUE', + 'lines': { + 1: 'url = {}'.format(url_format.format(second_secret)), + }, + 'secrets': second_secret, + 'types': [ + BasicAuthDetector.secret_type, + JwtTokenDetector.secret_type, + ], }, - 'secrets': second_secret, - 'types': [ - BasicAuthDetector.secret_type, - JwtTokenDetector.secret_type, - ], - }, - { - 'category': 'VERIFIED_FALSE', - 'lines': { - 2: 'example = {}'.format(url_format.format(random_secret)), + { + 'category': 'VERIFIED_FALSE', + 'lines': { + 2: 'example = {}'.format(url_format.format(random_secret)), + }, + 'secrets': random_secret, + 'types': [ + BasicAuthDetector.secret_type, + ], }, - 'secrets': random_secret, - 'types': [ - BasicAuthDetector.secret_type, - ], - }, - ], + { + 'category': 'VERIFIED_TRUE', + 'lines': { + 1: 'aws_access_key = {}'.format(aws_secret), + }, + 'secrets': aws_secret, + 'types': [ + AWSKeyDetector.secret_type, + ], + }, + ], + }, ), ( - SecretClassToPrint.REAL_SECRET, 3, 0, [ - { - 'category': 'VERIFIED_TRUE', - 'lines': { - 1: 'url = {}'.format(url_format.format(first_secret)), - 3: 'link = {}'.format(url_format.format(first_secret)), + SecretClassToPrint.REAL_SECRET, 4, 0, + { + 'results': [ + { + 'category': 'VERIFIED_TRUE', + 'lines': { + 1: 'url = {}'.format(url_format.format(first_secret)), + 3: 'link = {}'.format(url_format.format(first_secret)), + }, + 'secrets': first_secret, + 'types': [ + BasicAuthDetector.secret_type, + ], }, - 'secrets': first_secret, - 'types': [ - BasicAuthDetector.secret_type, - ], - }, - { - 'category': 'UNVERIFIED', - 'lines': { - 2: 'example = {}'.format(url_format.format(random_secret)), + { + 'category': 'UNVERIFIED', + 'lines': { + 2: 'example = {}'.format(url_format.format(random_secret)), + }, + 'secrets': random_secret, + 'types': [ + BasicAuthDetector.secret_type, + ], }, - 'secrets': random_secret, - 'types': [ - BasicAuthDetector.secret_type, - ], - }, - { - 'category': 'VERIFIED_TRUE', - 'lines': { - 1: 'url = {}'.format(url_format.format(second_secret)), + { + 'category': 'VERIFIED_TRUE', + 'lines': { + 1: 'url = {}'.format(url_format.format(second_secret)), + }, + 'secrets': second_secret, + 'types': [ + JwtTokenDetector.secret_type, + ], }, - 'secrets': second_secret, - 'types': [ - JwtTokenDetector.secret_type, - ], - }, - ], + { + 'category': 'VERIFIED_TRUE', + 'lines': { + 1: 'aws_access_key = {}'.format(aws_secret), + }, + 'secrets': aws_secret, + 'types': [ + AWSKeyDetector.secret_type, + ], + }, + ], + }, ), ( - SecretClassToPrint.FALSE_POSITIVE, 0, 2, [ - { - 'category': 'VERIFIED_FALSE', - 'lines': { - 1: 'url = {}'.format(url_format.format(second_secret)), + SecretClassToPrint.FALSE_POSITIVE, 0, 2, + { + 'results': [ + { + 'category': 'VERIFIED_FALSE', + 'lines': { + 1: 'url = {}'.format(url_format.format(second_secret)), + }, + 'secrets': second_secret, + 'types': [ + BasicAuthDetector.secret_type, + ], }, - 'secrets': second_secret, - 'types': [ - BasicAuthDetector.secret_type, - ], - }, - { - 'category': 'VERIFIED_FALSE', - 'lines': { - 2: 'example = {}'.format(url_format.format(random_secret)), + { + 'category': 'VERIFIED_FALSE', + 'lines': { + 2: 'example = {}'.format(url_format.format(random_secret)), + }, + 'secrets': random_secret, + 'types': [ + BasicAuthDetector.secret_type, + ], }, - 'secrets': random_secret, - 'types': [ - BasicAuthDetector.secret_type, - ], - }, - ], + ], + }, ), ], ) @@ -143,9 +174,9 @@ def test_generate_report( real, false = count_results(output) assert real == expected_real assert false == expected_false - for expected in expected_output: + for expected in expected_output['results']: found = False - for item in output: + for item in output['results']: if expected['secrets'] == item['secrets'] and expected['category'] == item['category']: for key in expected.keys(): assert item[key] == expected[key] @@ -156,7 +187,7 @@ def test_generate_report( def count_results(data): real_secrets = 0 false_secrets = 0 - for secret in data: + for secret in data['results']: if SecretClassToPrint.from_class(VerifiedResult[secret['category']]) == SecretClassToPrint.REAL_SECRET: # noqa: E501 real_secrets += 1 else: @@ -184,20 +215,33 @@ def baseline_file(): url = {url_format.format(second_secret)} example = {url_format.format(random_secret)} """)[1:] + third_content = textwrap.dedent(f""" + aws_access_key = {aws_secret} + """)[1:] with create_file_with_content(first_content) as first_file, \ create_file_with_content(second_content) as second_file, \ + create_file_with_content(third_content) as third_file, \ mock_named_temporary_file() as baseline_file, \ transient_settings({ 'plugins_used': [ {'name': 'BasicAuthDetector'}, {'name': 'JwtTokenDetector'}, + {'name': 'AWSKeyDetector'}, ], + 'filters_used': [ + { + 'path': + 'detect_secrets.filters.common.is_ignored_due_to_verification_policies', + 'min_level': 2, + }, + ], }): secrets = SecretsCollection() secrets.scan_file(first_file) secrets.scan_file(second_file) + secrets.scan_file(third_file) labels = { (first_file, BasicAuthDetector.secret_type, 1): True, (first_file, BasicAuthDetector.secret_type, 2): None, @@ -205,6 +249,7 @@ def baseline_file(): (second_file, JwtTokenDetector.secret_type, 1): True, (second_file, BasicAuthDetector.secret_type, 1): False, (second_file, BasicAuthDetector.secret_type, 2): False, + (third_file, AWSKeyDetector.secret_type, 1): True, } for item in secrets: _, secret = item diff --git a/tests/filters/heuristic_filter_test.py b/tests/filters/heuristic_filter_test.py index 3906a8feb..90e1eb0de 100644 --- a/tests/filters/heuristic_filter_test.py +++ b/tests/filters/heuristic_filter_test.py @@ -4,6 +4,7 @@ from detect_secrets import filters from detect_secrets.core.scan import scan_line +from detect_secrets.plugins.aws import AWSKeyDetector from detect_secrets.settings import transient_settings @@ -77,23 +78,26 @@ def test_success(self, secret, line): assert filters.heuristic.is_likely_id_string(secret, line) @pytest.mark.parametrize( - 'secret, line', + 'secret, line, plugin', [ # the word hidden has the word id in it, but lets # not mark that as an id string - ('RANDOM_STRING', 'hidden_secret: RANDOM_STRING'), - ('RANDOM_STRING', 'hidden_secret=RANDOM_STRING'), - ('RANDOM_STRING', 'hidden_secret = RANDOM_STRING'), + ('RANDOM_STRING', 'hidden_secret: RANDOM_STRING', None), + ('RANDOM_STRING', 'hidden_secret=RANDOM_STRING', None), + ('RANDOM_STRING', 'hidden_secret = RANDOM_STRING', None), # fail silently if the secret isn't even on the line - ('SOME_RANDOM_STRING', 'id: SOME_OTHER_RANDOM_STRING'), + ('SOME_RANDOM_STRING', 'id: SOME_OTHER_RANDOM_STRING', None), # fail although the word david ends in id - ('RANDOM_STRING', 'postgres://david:RANDOM_STRING'), + ('RANDOM_STRING', 'postgres://david:RANDOM_STRING', None), + + # fail since this is an aws access key id, a real secret + ('AKIA4NACSIJMDDNSEDTE', 'aws_access_key_id=AKIA4NACSIJMDDNSEDTE', AWSKeyDetector()), ], ) - def test_failure(self, secret, line): - assert not filters.heuristic.is_likely_id_string(secret, line) + def test_failure(self, secret, line, plugin): + assert not filters.heuristic.is_likely_id_string(secret, line, plugin) @pytest.mark.parametrize( @@ -117,9 +121,16 @@ def test_is_templated_secret(line, result): assert bool(list(scan_line(line))) is result -def test_is_prefixed_with_dollar_sign(): - assert filters.heuristic.is_prefixed_with_dollar_sign('$secret') - assert not filters.heuristic.is_prefixed_with_dollar_sign('secret') +@pytest.mark.parametrize( + 'secret, result', + ( + ('$secret', True), + ('secret', False), + ('', False), + ), +) +def test_is_prefixed_with_dollar_sign(secret, result): + assert filters.heuristic.is_prefixed_with_dollar_sign(secret) == result @pytest.mark.parametrize( @@ -135,8 +146,9 @@ def test_is_indirect_reference(line, result): def test_is_lock_file(): - # Basic test + # Basic tests assert filters.heuristic.is_lock_file('composer.lock') + assert filters.heuristic.is_lock_file('packages.lock.json') # file path assert filters.heuristic.is_lock_file('path/yarn.lock') diff --git a/tests/plugins/discord_test.py b/tests/plugins/discord_test.py new file mode 100644 index 000000000..dde4996c1 --- /dev/null +++ b/tests/plugins/discord_test.py @@ -0,0 +1,93 @@ +import pytest + +from detect_secrets.plugins.discord import DiscordBotTokenDetector + + +class TestDiscordBotTokenDetector: + + @pytest.mark.parametrize( + 'payload, should_flag', + [ + # From https://discord.com/developers/docs/reference#authentication + ( + 'MTk4NjIyNDgzNDcxOTI1MjQ4.Cl2FMQ.ZnCjm1XVW7vRze4b7Cq4se7kKWs', + True, + ), + ( + 'Nzk5MjgxNDk0NDc2NDU1OTg3.YABS5g.2lmzECVlZv3vv6miVnUaKPQi2wI', + True, + ), + # From https://docs.gitguardian.com/secrets-detection/detectors/specifics/discord_bot_token#examples # noqa: E501 + ( + 'MZ1yGvKTjE0rY0cV8i47CjAa.uRHQPq.Xb1Mk2nEhe-4iUcrGOuegj57zMC', + True, + ), + # From https://github.com/Yelp/detect-secrets/issues/627 + ( + 'OTUyNED5MDk2MTMxNzc2MkEz.YjESug.UNf-1GhsIG8zWT409q2C7Bh_zWQ', + True, + ), + ( + 'OTUyNED5MDk2MTMxNzc2MkEz.GSroKE.g2MTwve8OnUAAByz8KV_ZTV1Ipzg4o_NmQWUMs', + True, + ), + ( + 'MTAyOTQ4MTN5OTU5MTDwMEcxNg.GSwJyi.sbaw8msOR3Wi6vPUzeIWy_P0vJbB0UuRVjH8l8', + True, + ), + # Pass - token starts on the 3rd character (first segment is 24 characters) + ( + 'ATMyOTQ4MTN5OTU5MTDwMEcxNg.GSwJyi.sbaw8msOR3Wi6vPUzeIWy_P0vJbB0UuRVjH8l8', + True, + ), + # Pass - token starts on the 2nd character (first segment is 25 characters) + ( + '=MTAyOTQ4MTN5OTU5MTDwMEcxN.GSwJyi.sbaw8msOR3Wi6vPUzeIWy_P0vJbB0UuRVjH8l8', + True, + ), + # Pass - token ends before the '!' (last segment is 27 characters) + ( + 'MTAyOTQ4MTN5OTU5MTDwMEcxNg.YjESug.UNf-1GhsIG8zWT409q2C7Bh_zWQ!4o_NmQWUMs', + True, + ), + # Fail - all segments too short (23.5.26) + ( + 'MZ1yGvKTj0rY0cV8i47CjAa.uHQPq.Xb1Mk2nEhe-4icrGOuegj57zMC', + False, + ), + # Fail - first segment too short (23.6.27) + ( + 'MZ1yGvKTj0rY0cV8i47CjAa.uRHQPq.Xb1Mk2nEhe-4iUcrGOuegj57zMC', + False, + ), + # Fail - middle segment too short (24.5.27) + ( + 'MZ1yGvKTjE0rY0cV8i47CjAa.uHQPq.Xb1Mk2nEhe-4iUcrGOuegj57zMC', + False, + ), + # Fail - last segment too short (24.6.26) + ( + 'MZ1yGvKTjE0rY0cV8i47CjAa.uRHQPq.Xb1Mk2nEhe-4iUcrGOuegj57zM', + False, + ), + # Fail - contains invalid character ',' + ( + 'MZ1yGvKTjE0rY0cV8i47CjAa.uRHQPq.Xb1Mk2nEhe,4iUcrGOuegj57zMC', + False, + ), + # Fail - invalid first character 'P' (must be one of M/N/O) + ( + 'PZ1yGvKTjE0rY0cV8i47CjAa.uRHQPq.Xb1Mk2nEhe-4iUcrGOuegj57zMC', + False, + ), + # Fail - first segment 1 character too long; causes invalid first character 'T' + ( + 'MTAyOTQ4MTN5OTU5MTDwMEcxNg0.GSwJyi.sbaw8msOR3Wi6vPUzeIWy_P0vJbB0UuRVjH8l8', + False, + ), + ], + ) + def test_analyze(self, payload, should_flag): + logic = DiscordBotTokenDetector() + output = logic.analyze_line(filename='mock_filename', line=payload) + assert len(output) == (1 if should_flag else 0) diff --git a/tests/plugins/keyword_test.py b/tests/plugins/keyword_test.py index ec5cf4ce2..003d8dd38 100644 --- a/tests/plugins/keyword_test.py +++ b/tests/plugins/keyword_test.py @@ -83,6 +83,8 @@ ('password := "somefakekey"', None), # 'fake' in the secret ('some_key = "real_secret"', None), # We cannot make 'key' a Keyword, too noisy) ('private_key "hopenobodyfindsthisone\';', None), # Double-quote does not match single-quote) + ('password: real_key', None), + ('password: "real_key"', None), (LONG_LINE, None), # Long line test ] diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py index a7a595d72..84da3e335 100644 --- a/tests/pre_commit_hook_test.py +++ b/tests/pre_commit_hook_test.py @@ -114,11 +114,11 @@ def test_console_output_json_formatting(): # Assert formatting data = json.loads(capturedOutput.getvalue()) - assert(data['version']) - assert(data['plugins_used']) - assert(data['filters_used']) - assert(data['results']) - assert(data['generated_at']) + assert (data['version']) + assert (data['plugins_used']) + assert (data['filters_used']) + assert (data['results']) + assert (data['generated_at']) class TestModifiesBaselineFromVersionChange: diff --git a/tests/transformers/yaml_transformer_test.py b/tests/transformers/yaml_transformer_test.py index 9aa736ca2..5c0f467f9 100644 --- a/tests/transformers/yaml_transformer_test.py +++ b/tests/transformers/yaml_transformer_test.py @@ -121,6 +121,43 @@ def test_multi_line_flow_mapping(): 'keyD: "valueD"', ] + @staticmethod + def test_single_anchor_tag(): + file = mock_file_object( + textwrap.dedent(""" + keyA: &test + keyB: string # with comments + """)[1:-1], + ) + + assert YAMLTransformer().parse_file(file) == [ + '', + 'keyB: "string" # with comments', + ] + + @staticmethod + def test_anchor_tag_alias_combination(): + file = mock_file_object( + textwrap.dedent(""" + groupA: &groupA + keyA: valueA + keyB: valueB + + groupB: &groupB + keyC: valueC + keyD: *groupA + """)[1:-1], + ) + + assert YAMLTransformer().parse_file(file) == [ + '', + 'keyA: "valueA"', + 'keyB: "valueB"', + '', + '', + 'keyC: "valueC"', + ] + class TestYAMLFileParser: @staticmethod @@ -428,3 +465,80 @@ def test_inline_mapping_single_line_multikey_line_numbers(): '__original_key__': 'd', }, } + + @staticmethod + def test_single_anchor_tag(): + file = mock_file_object( + textwrap.dedent(""" + keyA: &test + keyB: string # with comments + keyC: + keyD: string + """)[1:-1], + ) + + assert YAMLFileParser(file).json() == { + 'keyA': { + 'keyB': { + '__value__': 'string', + '__line__': 2, + '__original_key__': 'keyB', + }, + 'keyC': { + 'keyD': { + '__value__': 'string', + '__line__': 4, + '__original_key__': 'keyD', + }, + }, + }, + } + + @staticmethod + def test_anchor_tag_alias_combination(): + file = mock_file_object( + textwrap.dedent(""" + groupA: &groupA + keyA: valueA + keyB: valueB + + groupB: &groupB + keyC: valueC + keyD: *groupA + """)[1:-1], + ) + + temp = YAMLFileParser(file).json() + assert temp == { + 'groupA': { + 'keyA': { + '__value__': 'valueA', + '__line__': 2, + '__original_key__': 'keyA', + }, + 'keyB': { + '__value__': 'valueB', + '__line__': 3, + '__original_key__': 'keyB', + }, + }, + 'groupB': { + 'keyC': { + '__value__': 'valueC', + '__line__': 6, + '__original_key__': 'keyC', + }, + 'keyD': { + 'keyA': { + '__value__': 'valueA', + '__line__': 2, + '__original_key__': 'keyA', + }, + 'keyB': { + '__value__': 'valueB', + '__line__': 3, + '__original_key__': 'keyB', + }, + }, + }, + } diff --git a/tox.ini b/tox.ini index 7c0e4a9bb..01f5d4d07 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,13 @@ [tox] project = detect_secrets # These should match the ci python env list -envlist = py{36,37,38,39},mypy +envlist = py{38,39,310,311},mypy skip_missing_interpreters = true -tox_pip_extensions_ext_venv_update = true [testenv] passenv = SSH_AUTH_SOCK # NO_PROXY is needed to call requests API within a forked process -# when using macOS and python version 3.6/3.7 +# when using macOS and python version 3.7 setenv = NO_PROXY = '*' deps = -rrequirements-dev.txt