Merge branch 'grafana:main' into main

.github/workflows/deploy-pr-preview.yml

@@ -0,0 +1,24 @@
+name: Deploy pr preview
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - closed
+    paths:
+      - "docs/sources/**"
+
+jobs:
+  deploy-pr-preview:
+    if: github.repository == 'grafana/alloy'
+    uses: grafana/writers-toolkit/.github/workflows/deploy-preview.yml@main
+    with:
+      sha: ${{ github.event.pull_request.head.sha }}
+      branch: ${{ github.head_ref }}
+      event_number: ${{ github.event.number }}
+      title: ${{ github.event.pull_request.title }}
+      repo: alloy
+      website_directory: content/docs/alloy/latest
+      relative_prefix: /docs/alloy/latest/
+      index_file: true

.github/workflows/helm-test.yml

@@ -71,7 +71,7 @@ jobs:
         run: ct lint --config ./operations/helm/ct.yaml
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.10.0
+        uses: helm/kind-action@v1.11.0
         if: steps.list-changed.outputs.changed == 'true'
 
       - name: Add dependency chart repos

CHANGELOG.md

@@ -50,11 +50,15 @@ Main (unreleased)
   - Change processlist query to support ONLY_FULL_GROUP_BY sql_mode
   - Add perf_schema quantile columns to collector
 
+- Live Debugging button should appear in UI only for supported components (@ravishankar15)
 - Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15)
+- Add `ignore_older_than` option for local.file_match (@ravishankar15)
+- Add livedebugging support for `discover.relabel` (@ravishankar15)
+- Performance optimization for live debugging feature (@ravishankar15)
 
-- Use a forked `github.com/goccy/go-json` module which reduces the memory consumption of an Alloy instance by 20MB.
+- Upgrade `github.com/goccy/go-json` to v0.10.4, which reduces the memory consumption of an Alloy instance by 20MB.
   If Alloy is running certain otelcol components, this reduction will not apply. (@ptodev)
-  
+
 - Update `prometheus.write.queue` library for performance increases in cpu. (@mattdurham)
 
 ### Bugfixes
@@ -105,7 +109,7 @@ v1.5.1
 
 - Fixed a crash when updating the configuration of `remote.http`. (@kinolaev)
 
-- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum) 
+- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum)
 
 - Fixed an issue in the `prometheus.exporter.postgres` component that would leak goroutines when the target was not reachable (@dehaansa)
 
@@ -305,7 +309,7 @@ v1.4.0
 
 - Add the label `alloy_cluster` in the metric `alloy_config_hash` when the flag `cluster.name` is set to help differentiate between
   configs from the same alloy cluster or different alloy clusters. (@wildum)
-  
+
 - Add support for discovering the cgroup path(s) of a process in `process.discovery`. (@mahendrapaipuri)
 
 ### Bugfixes

CODEOWNERS

@@ -9,6 +9,7 @@
 * @grafana/grafana-alloy-maintainers
 
 #`make docs` procedure and related workflows are owned by @jdbaldry.
+/.github/workflows/deploy-pr-preview.yml                       @jdbaldry
 /.github/workflows/publish-technical-documentation-next.yml    @jdbaldry
 /.github/workflows/publish-technical-documentation-release.yml @jdbaldry
 /.github/workflows/update-make-docs.yml                        @jdbaldry

docs/sources/reference/components/local/local.file_match.md

@@ -24,16 +24,19 @@ local.file_match "LABEL" {
 
 The following arguments are supported:
 
-Name            | Type                | Description                                                                                | Default | Required
---------------- | ------------------- | ------------------------------------------------------------------------------------------ |---------| --------
-`path_targets`  | `list(map(string))` | Targets to expand; looks for glob patterns on the  `__path__` and `__path_exclude__` keys. |         | yes
-`sync_period`   | `duration`          | How often to sync filesystem and targets.                                                  | `"10s"` | no
+Name                | Type                | Description                                                                                | Default | Required
+---------------     | ------------------- | ------------------------------------------------------------------------------------------ |---------| --------
+`path_targets`      | `list(map(string))` | Targets to expand; looks for glob patterns on the  `__path__` and `__path_exclude__` keys. |         | yes
+`sync_period`       | `duration`          | How often to sync filesystem and targets.                                                  | `"10s"` | no
+`ignore_older_than` | `duration`          | Ignores files which are modified before this duration.                                     |  `"0s"` | no
 
 `path_targets` uses [doublestar][] style paths.
 * `/tmp/**/*.log` will match all subfolders of `tmp` and include any files that end in `*.log`.
 * `/tmp/apache/*.log` will match only files in `/tmp/apache/` that end in `*.log`.
 * `/tmp/**` will match all subfolders of `tmp`, `tmp` itself, and all files.
 
+`local.file_match` doesn't ignore files when `ignore_older_than` is set to the default, `0s`.
+
 
 ## Exported fields
 

docs/sources/reference/components/prometheus/prometheus.remote_write.md

@@ -427,7 +427,7 @@ To troubleshoot, take the following steps in order:
    You can use [Promtool][promtool] to inspect it and find out which metric series were sent by this {{< param "PRODUCT_NAME" >}} instance since the last WAL truncation event. 
    For example:
    ```
-   ./promtool tsdb dump --match='{__name__=\"otelcol_connector_spanmetrics_duration_seconds_bucket\", http_method=\"GET\", job=\"ExampleJobName\"' /path/to/wal/ 
+   ./promtool tsdb dump --match='{__name__="otelcol_connector_spanmetrics_duration_seconds_bucket", http_method="GET", job="ExampleJobName"}' /path/to/wal/
    ```
 
 [clustering]: ../../configure/clustering

docs/sources/reference/config-blocks/livedebugging.md

@@ -30,8 +30,8 @@ livedebugging {
 
 The following arguments are supported:
 
-| Name      | Type   | Description                         | Default | Required |
-| --------- | ------ | ----------------------------------- | ------- | -------- |
-| `enabled` | `bool` | Enables the live debugging feature. | `false` | no       |
+| Name                 | Type  | Description                                                      | Default | Required |
+| -------------------- | ----- | ---------------------------------------------------------------  | ------- | -------- |
+| `enabled`            | `bool`| Enables the live debugging feature.                              | `false` | no       |
 
 [debug]: ../../../troubleshoot/debug/

docs/sources/troubleshoot/debug.md

@@ -111,6 +111,7 @@ Supported components:
 * `otelcol.receiver.*`
 * `prometheus.relabel`
 {{< /admonition >}}
+* `discovery.relabel`
 
 ## Debug using the UI
 

go.mod

@@ -518,7 +518,7 @@ require (
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/goccy/go-json v0.10.4 // indirect
 	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
@@ -945,6 +945,3 @@ exclude (
 )
 
 replace github.com/prometheus/procfs => github.com/prometheus/procfs v0.12.0
-
-// TODO(ptodev): Remove when this PR has been merged: https://github.com/goccy/go-json/pull/490
-replace github.com/goccy/go-json => github.com/grafana/go-json v0.0.0-20241106155216-71a03f133f5c

go.sum

@@ -1619,6 +1619,9 @@ github.com/goburrow/modbus v0.1.0/go.mod h1:Kx552D5rLIS8E7TyUwQ/UdHEqvX5T8tyiGBT
 github.com/goburrow/serial v0.1.0/go.mod h1:sAiqG0nRVswsm1C97xsttiYCzSLBmUZ/VSlVLZJ8haA=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
+github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -1853,8 +1856,6 @@ github.com/grafana/dskit v0.0.0-20240104111617-ea101a3b86eb h1:AWE6+kvtE18HP+lRW
 github.com/grafana/dskit v0.0.0-20240104111617-ea101a3b86eb/go.mod h1:kkWM4WUV230bNG3urVRWPBnSJHs64y/0RmWjftnnn0c=
 github.com/grafana/go-gelf/v2 v2.0.1 h1:BOChP0h/jLeD+7F9mL7tq10xVkDG15he3T1zHuQaWak=
 github.com/grafana/go-gelf/v2 v2.0.1/go.mod h1:lexHie0xzYGwCgiRGcvZ723bSNyNI8ZRD4s0CLobh90=
-github.com/grafana/go-json v0.0.0-20241106155216-71a03f133f5c h1:yKBKEC347YZpgii1KazRCfxHsTaxMqWZzoivM1OTT50=
-github.com/grafana/go-json v0.0.0-20241106155216-71a03f133f5c/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/grafana/go-offsets-tracker v0.1.7 h1:2zBQ7iiGzvyXY7LA8kaaSiEqH/Yx82UcfRabbY5aOG4=
 github.com/grafana/go-offsets-tracker v0.1.7/go.mod h1:qcQdu7zlUKIFNUdBJlLyNHuJGW0SKWKjkrN6jtt+jds=
 github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85/go.mod h1:crI9WX6p0IhrqB+DqIUHulRW853PaNFf7o4UprV//3I=

internal/cmd/integration-tests/docker-compose.yaml

@@ -12,7 +12,7 @@ services:
       - "9009:9009"
 
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.6.1
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./configs/tempo/tempo.yaml:/etc/tempo.yaml

internal/component/component_provider.go

@@ -123,9 +123,10 @@ type Info struct {
 	ComponentName string // Name of the component.
 	Health        Health // Current component health.
 
-	Arguments Arguments   // Current arguments value of the component.
-	Exports   Exports     // Current exports value of the component.
-	DebugInfo interface{} // Current debug info of the component.
+	Arguments            Arguments   // Current arguments value of the component.
+	Exports              Exports     // Current exports value of the component.
+	DebugInfo            interface{} // Current debug info of the component.
+	LiveDebuggingEnabled bool
 }
 
 // MarshalJSON returns a JSON representation of cd. The format of the
@@ -139,19 +140,20 @@ func (info *Info) MarshalJSON() ([]byte, error) {
 		}
 
 		componentDetailJSON struct {
-			Name             string               `json:"name"`
-			Type             string               `json:"type,omitempty"`
-			LocalID          string               `json:"localID"`
-			ModuleID         string               `json:"moduleID"`
-			Label            string               `json:"label,omitempty"`
-			References       []string             `json:"referencesTo"`
-			ReferencedBy     []string             `json:"referencedBy"`
-			Health           *componentHealthJSON `json:"health"`
-			Original         string               `json:"original"`
-			Arguments        json.RawMessage      `json:"arguments,omitempty"`
-			Exports          json.RawMessage      `json:"exports,omitempty"`
-			DebugInfo        json.RawMessage      `json:"debugInfo,omitempty"`
-			CreatedModuleIDs []string             `json:"createdModuleIDs,omitempty"`
+			Name                 string               `json:"name"`
+			Type                 string               `json:"type,omitempty"`
+			LocalID              string               `json:"localID"`
+			ModuleID             string               `json:"moduleID"`
+			Label                string               `json:"label,omitempty"`
+			References           []string             `json:"referencesTo"`
+			ReferencedBy         []string             `json:"referencedBy"`
+			Health               *componentHealthJSON `json:"health"`
+			Original             string               `json:"original"`
+			Arguments            json.RawMessage      `json:"arguments,omitempty"`
+			Exports              json.RawMessage      `json:"exports,omitempty"`
+			DebugInfo            json.RawMessage      `json:"debugInfo,omitempty"`
+			CreatedModuleIDs     []string             `json:"createdModuleIDs,omitempty"`
+			LiveDebuggingEnabled bool                 `json:"liveDebuggingEnabled"`
 		}
 	)
 
@@ -196,10 +198,11 @@ func (info *Info) MarshalJSON() ([]byte, error) {
 			Message:     info.Health.Message,
 			UpdatedTime: info.Health.UpdateTime,
 		},
-		Arguments:        arguments,
-		Exports:          exports,
-		DebugInfo:        debugInfo,
-		CreatedModuleIDs: info.ModuleIDs,
+		Arguments:            arguments,
+		Exports:              exports,
+		DebugInfo:            debugInfo,
+		CreatedModuleIDs:     info.ModuleIDs,
+		LiveDebuggingEnabled: info.LiveDebuggingEnabled,
 	})
 }
 

internal/component/discovery/relabel/relabel.go

@@ -2,12 +2,14 @@ package relabel
 
 import (
 	"context"
+	"fmt"
 	"sync"
 
 	"github.com/grafana/alloy/internal/component"
 	alloy_relabel "github.com/grafana/alloy/internal/component/common/relabel"
 	"github.com/grafana/alloy/internal/component/discovery"
 	"github.com/grafana/alloy/internal/featuregate"
+	"github.com/grafana/alloy/internal/service/livedebugging"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
 )
@@ -46,13 +48,23 @@ type Component struct {
 
 	mut sync.RWMutex
 	rcs []*relabel.Config
+
+	debugDataPublisher livedebugging.DebugDataPublisher
 }
 
 var _ component.Component = (*Component)(nil)
+var _ component.LiveDebugging = (*Component)(nil)
 
 // New creates a new discovery.relabel component.
 func New(o component.Options, args Arguments) (*Component, error) {
-	c := &Component{opts: o}
+	debugDataPublisher, err := o.GetServiceData(livedebugging.ServiceName)
+	if err != nil {
+		return nil, err
+	}
+	c := &Component{
+		opts:               o,
+		debugDataPublisher: debugDataPublisher.(livedebugging.DebugDataPublisher),
+	}
 
 	// Call to Update() to set the output once at the start
 	if err := c.Update(args); err != nil {
@@ -81,9 +93,13 @@ func (c *Component) Update(args component.Arguments) error {
 
 	for _, t := range newArgs.Targets {
 		lset := componentMapToPromLabels(t)
-		lset, keep := relabel.Process(lset, relabelConfigs...)
+		relabelled, keep := relabel.Process(lset, relabelConfigs...)
 		if keep {
-			targets = append(targets, promLabelsToComponent(lset))
+			targets = append(targets, promLabelsToComponent(relabelled))
+		}
+		componentID := livedebugging.ComponentID(c.opts.ID)
+		if c.debugDataPublisher.IsActive(componentID) {
+			c.debugDataPublisher.Publish(componentID, fmt.Sprintf("%s => %s", lset.String(), relabelled.String()))
 		}
 	}
 
@@ -95,6 +111,8 @@ func (c *Component) Update(args component.Arguments) error {
 	return nil
 }
 
+func (c *Component) LiveDebugging(_ int) {}
+
 func componentMapToPromLabels(ls discovery.Target) labels.Labels {
 	res := make([]labels.Label, 0, len(ls))
 	for k, v := range ls {

internal/component/local/file_match/file.go

@@ -26,8 +26,9 @@ func init() {
 // Arguments holds values which are used to configure the local.file_match
 // component.
 type Arguments struct {
-	PathTargets []discovery.Target `alloy:"path_targets,attr"`
-	SyncPeriod  time.Duration      `alloy:"sync_period,attr,optional"`
+	PathTargets     []discovery.Target `alloy:"path_targets,attr"`
+	SyncPeriod      time.Duration      `alloy:"sync_period,attr,optional"`
+	IgnoreOlderThan time.Duration      `alloy:"ignore_older_than,attr,optional"`
 }
 
 var _ component.Component = (*Component)(nil)
@@ -80,8 +81,9 @@ func (c *Component) Update(args component.Arguments) error {
 	c.watches = c.watches[:0]
 	for _, v := range c.args.PathTargets {
 		c.watches = append(c.watches, watch{
-			target: v,
-			log:    c.opts.Logger,
+			target:          v,
+			log:             c.opts.Logger,
+			ignoreOlderThan: c.args.IgnoreOlderThan,
 		})
 	}
 

internal/component/local/file_match/file_test.go

@@ -63,6 +63,35 @@ func TestDirectoryFile(t *testing.T) {
 	require.True(t, contains(foundFiles, "t1.txt"))
 }
 
+func TestFileIgnoreOlder(t *testing.T) {
+	dir := path.Join(os.TempDir(), "alloy_testing", "t1")
+	err := os.MkdirAll(dir, 0755)
+	require.NoError(t, err)
+	writeFile(t, dir, "t1.txt")
+	t.Cleanup(func() {
+		os.RemoveAll(dir)
+	})
+	c := createComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil)
+	ct := context.Background()
+	ct, ccl := context.WithTimeout(ct, 5*time.Second)
+	defer ccl()
+	c.args.SyncPeriod = 10 * time.Millisecond
+	c.args.IgnoreOlderThan = 100 * time.Millisecond
+	c.Update(c.args)
+	go c.Run(ct)
+
+	foundFiles := c.getWatchedFiles()
+	require.Len(t, foundFiles, 1)
+	require.True(t, contains(foundFiles, "t1.txt"))
+	time.Sleep(150 * time.Millisecond)
+
+	writeFile(t, dir, "t2.txt")
+	ct.Done()
+	foundFiles = c.getWatchedFiles()
+	require.Len(t, foundFiles, 1)
+	require.True(t, contains(foundFiles, "t2.txt"))
+}
+
 func TestAddingFile(t *testing.T) {
 	dir := path.Join(os.TempDir(), "alloy_testing", "t2")
 	err := os.MkdirAll(dir, 0755)

internal/component/local/file_match/watch.go

@@ -3,6 +3,7 @@ package file_match
 import (
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/bmatcuk/doublestar"
 	"github.com/go-kit/log"
@@ -14,8 +15,9 @@ import (
 
 // watch handles a single discovery.target for file watching.
 type watch struct {
-	target discovery.Target
-	log    log.Logger
+	target          discovery.Target
+	log             log.Logger
+	ignoreOlderThan time.Duration
 }
 
 func (w *watch) getPaths() ([]discovery.Target, error) {
@@ -48,9 +50,15 @@ func (w *watch) getPaths() ([]discovery.Target, error) {
 			}
 			continue
 		}
+
 		if fi.IsDir() {
 			continue
 		}
+
+		if w.ignoreOlderThan != 0 && fi.ModTime().Before(time.Now().Add(-w.ignoreOlderThan)) {
+			continue
+		}
+
 		dt := discovery.Target{}
 		for dk, v := range w.target {
 			dt[dk] = v