fix(loki.secretfilter): Fix partial masking for short secrets and sup…

…port multiple allowlists per rule (#2320)

* Fix partial masking bug and support new allowlist format

* Add docs and changelog

* Update docs

* Add comments

* Add comments

* Minor docs update

* Add more tests

* Change criteria for partial redaction

* Changes to partial masking rules

* Fix comment location

* Clarify usage of secret types

* Clarify usage of secret types

* Update docs/sources/reference/components/loki/loki.secretfilter.md

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

* Suggestions

* Suggestions

---------

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

CHANGELOG.md

@@ -81,6 +81,8 @@ v1.6.0-rc.0
 
 - Update `prometheus.write.queue` library for performance increases in cpu. (@mattdurham)
 
+- Update `loki.secretfilter` to be compatible with the new `[[rules.allowlists]]` gitleaks allowlist format (@romain-gaillard)
+
 - Update `async-profiler` binaries for `pyroscope.java` to 3.0-fa937db (@aleks-p)
 
 - Reduced memory allocation in discovery components by up to 30% (@thampiotr)
@@ -97,6 +99,8 @@ v1.6.0-rc.0
 
 - Fixed an issue where the `otelcol.processor.interval` could not be used because the debug metrics were not set to default. (@wildum)
 
+- Fixed an issue where `loki.secretfilter` would crash if the secret was shorter than the `partial_mask` value. (@romain-gaillard)
+
 - Change the log level in the `eventlogmessage` stage of the `loki.process` component from `warn` to `debug`. (@wildum)
 
 - Fix a bug in `loki.source.kafka` where the `topics` argument incorrectly used regex matching instead of exact matches. (@wildum)

docs/sources/reference/components/loki/loki.secretfilter.md

@@ -18,7 +18,7 @@ The detection is based on regular expression patterns, defined in the [Gitleaks
 
 {{< admonition type="caution" >}}
 Personally Identifiable Information (PII) isn't currently in scope and some secrets could remain undetected.
-This component may generate false positives.
+This component may generate false positives or redact too much.
 Don't rely solely on this component to redact sensitive information.
 {{< /admonition >}}
 
@@ -49,7 +49,14 @@ Name                     | Type                 | Description
 The `gitleaks_config` argument is the path to the custom `gitleaks.toml` file.
 The Gitleaks configuration file embedded in the component is used if you don't provide the path to a custom configuration file.
 
-The `types` argument is a map of secret types to look for. The values are used as prefixes for the secret types in the Gitleaks configuration. If you don't provide this argument, all types are used.
+{{< admonition type="note" >}}
+This component doesn't support all the features of the Gitleaks configuration file. It only supports regular expression-based rules, `secretGroup`, and allowlist regular expressions. `regexTarget` only supports the default value `secret`. Other features such as `keywords`, `entropy`, `paths`, and `stopwords` are not supported. The `extend` feature is not supported. If you use a custom configuration file, you must include all the rules you want to use within the configuration file. Unsupported fields and values in the configuration file are ignored.
+{{< /admonition >}}
+
+The `types` argument is a map of secret types to look for.
+The values provided are used as prefixes to match rules IDs in the Gitleaks configuration.
+For example,  providing the type `grafana` will match the rules `grafana-api-key`, `grafana-cloud-api-token`, and `grafana-service-account-token`.
+If you don't provide this argument, all rules are used.
 
 {{< admonition type="note" >}}
 Configuring this argument with the secret types you want to look for is strongly recommended.
@@ -74,6 +81,8 @@ A secret will not be redacted if it matches any of the regular expressions. The
 
 The `partial_mask` argument is the number of characters to show from the beginning of the secret before the redact string is added.
 If set to `0`, the entire secret is redacted.
+If a secret is not at least 6 characters long, it will be entirely redacted.
+For short secrets, at most half of the secret is shown.
 
 ## Blocks
 

internal/component/loki/secretfilter/secretfilter.go

@@ -29,7 +29,6 @@ type Rule struct {
 	name        string
 	regex       *regexp.Regexp
 	secretGroup int
-	description string
 	allowlist   []AllowRule
 }
 
@@ -98,19 +97,20 @@ type Component struct {
 type GitLeaksConfig struct {
 	AllowList struct {
 		Description string
-		Paths       []string
 		Regexes     []string
 	}
 	Rules []struct {
 		ID          string
-		Description string
 		Regex       string
-		Keywords    []string
 		SecretGroup int
 
+		// Old format, kept for compatibility
 		Allowlist struct {
-			StopWords []string
-			Regexes   []string
+			Regexes []string
+		}
+		// New format
+		Allowlists []struct {
+			Regexes []string
 		}
 	}
 }
@@ -194,16 +194,16 @@ func (c *Component) processEntry(entry loki.Entry) loki.Entry {
 
 			// Check if the secret is in the allowlist
 			var allowRule *AllowRule = nil
-			// First check the rule-specific allowlist
-			for _, a := range r.allowlist {
+			// First check the global allowlist
+			for _, a := range c.AllowList {
 				if a.Regex.MatchString(secret) {
 					allowRule = &a
 					break
 				}
 			}
-			// Then check the global allowlist
+			// Then check the rule-specific allowlists
 			if allowRule == nil {
-				for _, a := range c.AllowList {
+				for _, a := range r.allowlist {
 					if a.Regex.MatchString(secret) {
 						allowRule = &a
 						break
@@ -231,8 +231,21 @@ func (c *Component) redactLine(line string, secret string, ruleName string) stri
 		redactWith = strings.ReplaceAll(redactWith, "$SECRET_NAME", ruleName)
 		redactWith = strings.ReplaceAll(redactWith, "$SECRET_HASH", hashSecret(secret))
 	}
-	if c.args.PartialMask > 0 {
-		redactWith = secret[:c.args.PartialMask] + redactWith
+
+	// If partialMask is set, show the first N characters of the secret
+	partialMask := int(c.args.PartialMask)
+	if partialMask < 0 {
+		partialMask = 0
+	}
+	runesSecret := []rune(secret)
+	// Only do it if the secret is long enough
+	if partialMask > 0 && len(runesSecret) >= 6 {
+		// Show at most half of the secret
+		if partialMask > len(runesSecret)/2 {
+			partialMask = len(runesSecret) / 2
+		}
+		prefix := string(runesSecret[:partialMask])
+		redactWith = prefix + redactWith
 	}
 
 	line = strings.ReplaceAll(line, secret, redactWith)
@@ -306,12 +319,21 @@ func (c *Component) Update(args component.Arguments) error {
 			}
 			allowlist = append(allowlist, AllowRule{Regex: re, Source: fmt.Sprintf("rule %s", rule.ID)})
 		}
+		for _, currAllowList := range rule.Allowlists {
+			for _, r := range currAllowList.Regexes {
+				re, err := regexp.Compile(r)
+				if err != nil {
+					level.Error(c.opts.Logger).Log("msg", "error compiling allowlist regex", "error", err)
+					return err
+				}
+				allowlist = append(allowlist, AllowRule{Regex: re, Source: fmt.Sprintf("rule %s", rule.ID)})
+			}
+		}
 
 		newRule := Rule{
 			name:        rule.ID,
 			regex:       re,
 			secretGroup: rule.SecretGroup,
-			description: rule.Description,
 			allowlist:   allowlist,
 		}
 
@@ -325,23 +347,23 @@ func (c *Component) Update(args component.Arguments) error {
 	}
 
 	// Compiling global allowlist regexes
-	// From the Gitleaks config
-	for _, r := range gitleaksCfg.AllowList.Regexes {
+	// From the arguments
+	for _, r := range c.args.AllowList {
 		re, err := regexp.Compile(r)
 		if err != nil {
 			level.Error(c.opts.Logger).Log("msg", "error compiling allowlist regex", "error", err)
 			return err
 		}
-		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "gitleaks config"})
+		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "alloy config"})
 	}
-	// From the arguments
-	for _, r := range c.args.AllowList {
+	// From the Gitleaks config
+	for _, r := range gitleaksCfg.AllowList.Regexes {
 		re, err := regexp.Compile(r)
 		if err != nil {
 			level.Error(c.opts.Logger).Log("msg", "error compiling allowlist regex", "error", err)
 			return err
 		}
-		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "alloy config"})
+		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "gitleaks config"})
 	}
 
 	// Add the generic API key rule last if needed