Restore code for drop_original_metrics and move its logic to the tran…

…slator (#340)
aws · Jul 18, 2023 · 6c2d68d · 6c2d68d
1 parent 3493d71
commit 6c2d68d
Show file tree

Hide file tree

Showing 13 changed files with 277 additions and 63 deletions.
diff --git a/plugins/outputs/cloudwatch/cloudwatch.go b/plugins/outputs/cloudwatch/cloudwatch.go
@@ -47,9 +47,8 @@ const (
 )
 
 const (
-	opPutLogEvents       = "PutLogEvents"
-	opPutMetricData      = "PutMetricData"
-	dropOriginalWildcard = "*"
+	opPutLogEvents  = "PutLogEvents"
+	opPutMetricData = "PutMetricData"
 )
 
 type CloudWatch struct {
@@ -392,9 +391,12 @@ func (c *CloudWatch) BuildMetricDatum(metric *aggregationDatum) []*cloudwatch.Me
 	}
 
 	dimensionsList := c.ProcessRollup(metric.Dimensions)
-	for _, dimensions := range dimensionsList {
+	for index, dimensions := range dimensionsList {
 		//index == 0 means it's the original metrics, and if the metric name and dimension matches, skip creating
 		//metric datum
+		if index == 0 && c.IsDropping(*metric.MetricDatum.MetricName) {
+			continue
+		}
 		if len(distList) == 0 {
 			// Not a distribution.
 			datum := &cloudwatch.MetricDatum{
@@ -434,6 +436,17 @@ func (c *CloudWatch) BuildMetricDatum(metric *aggregationDatum) []*cloudwatch.Me
 	return datums
 }
 
+func (c *CloudWatch) IsDropping(metricName string) bool {
+	// Check if any metrics are provided in drop_original_metrics
+	if len(c.config.DropOriginalConfigs) == 0 {
+		return false
+	}
+	if _, ok := c.config.DropOriginalConfigs[metricName]; ok {
+		return true
+	}
+	return false
+}
+
 // sortedTagKeys returns a sorted list of keys in the map.
 // Necessary for comparing a metric-name and its dimensions to determine
 // if 2 metrics are actually the same.

diff --git a/plugins/outputs/cloudwatch/cloudwatch_test.go b/plugins/outputs/cloudwatch/cloudwatch_test.go
@@ -19,6 +19,7 @@ import (
 	"github.com/influxdata/telegraf/metric"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 
 	"github.com/aws/private-amazon-cloudwatch-agent-staging/handlers/agentinfo"
 	"github.com/aws/private-amazon-cloudwatch-agent-staging/internal/publisher"
@@ -255,6 +256,66 @@ func TestGetUniqueRollupList(t *testing.T) {
 	assert.EqualValues(t, expectedLists, actualLists, "Unique list result should be empty")
 }
 
+func TestIsDropping(t *testing.T) {
+	svc := new(mockCloudWatchClient)
+	cw := newCloudWatchClient(svc, time.Second)
+
+	testCases := map[string]struct {
+		dropMetricsConfig    map[string]bool
+		expectMetricsDropped map[string]bool
+	}{
+		"TestIsDroppingWithMultipleCategoryLinux": {
+			dropMetricsConfig: map[string]bool{
+				"cpu_usage_idle":             true,
+				"cpu_time_active":            true,
+				"nvidia_smi_utilization_gpu": true,
+			},
+			expectMetricsDropped: map[string]bool{
+				"cpu_usage_idle":  true,
+				"cpu_time_active": true,
+				"nvidia_smi":      false,
+				"cpu_usage_guest": false,
+			},
+		},
+		"TestIsDroppingWithMultipleCategoryWindows": {
+			dropMetricsConfig: map[string]bool{
+				"cpu usage_idle":             true,
+				"cpu time_active":            true,
+				"nvidia_smi utilization_gpu": true,
+			},
+			expectMetricsDropped: map[string]bool{
+				"cpu usage_idle":  true,
+				"cpu time_active": true,
+				"nvidia_smi":      false,
+				"cpu usage_guest": false,
+			},
+		},
+		"TestIsDroppingWithMetricDecoration": {
+			dropMetricsConfig: map[string]bool{
+				"CPU_USAGE_IDLE":             true,
+				"cpu_time_active":            true,
+				"nvidia_smi_utilization_gpu": true,
+			},
+			expectMetricsDropped: map[string]bool{
+				"cpu_usage_idle":             false,
+				"CPU_USAGE_IDLE":             true,
+				"nvidia_smi":                 false,
+				"nvidia_smi_utilization_gpu": true,
+				"cpu":                        false,
+			},
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			cw.config.DropOriginalConfigs = testCase.dropMetricsConfig
+			for metricName, expectMetricDropped := range testCase.expectMetricsDropped {
+				actualMetricDropped := cw.IsDropping(metricName)
+				require.Equal(t, expectMetricDropped, actualMetricDropped)
+			}
+		})
+	}
+}
+
 func TestIsFlushable(t *testing.T) {
 	svc := new(mockCloudWatchClient)
 	res := cloudwatch.PutMetricDataOutput{}

diff --git a/plugins/outputs/cloudwatch/config.go b/plugins/outputs/cloudwatch/config.go
@@ -13,20 +13,20 @@ import (
 
 // Config represent a configuration for the CloudWatch logs exporter.
 type Config struct {
-	Region                   string              `mapstructure:"region"`
-	EndpointOverride         string              `mapstructure:"endpoint_override,omitempty"`
-	AccessKey                string              `mapstructure:"access_key,omitempty"`
-	SecretKey                string              `mapstructure:"secret_key,omitempty"`
-	RoleARN                  string              `mapstructure:"role_arn,omitempty"`
-	Profile                  string              `mapstructure:"profile,omitempty"`
-	SharedCredentialFilename string              `mapstructure:"shared_credential_file,omitempty"`
-	Token                    string              `mapstructure:"token,omitempty"`
-	ForceFlushInterval       time.Duration       `mapstructure:"force_flush_interval"`
-	MaxDatumsPerCall         int                 `mapstructure:"max_datums_per_call"`
-	MaxValuesPerDatum        int                 `mapstructure:"max_values_per_datum"`
-	RollupDimensions         [][]string          `mapstructure:"rollup_dimensions,omitempty"`
-	DropOriginConfigs        map[string][]string `mapstructure:"drop_original_metrics,omitempty"`
-	Namespace                string              `mapstructure:"namespace"`
+	Region                   string          `mapstructure:"region"`
+	EndpointOverride         string          `mapstructure:"endpoint_override,omitempty"`
+	AccessKey                string          `mapstructure:"access_key,omitempty"`
+	SecretKey                string          `mapstructure:"secret_key,omitempty"`
+	RoleARN                  string          `mapstructure:"role_arn,omitempty"`
+	Profile                  string          `mapstructure:"profile,omitempty"`
+	SharedCredentialFilename string          `mapstructure:"shared_credential_file,omitempty"`
+	Token                    string          `mapstructure:"token,omitempty"`
+	ForceFlushInterval       time.Duration   `mapstructure:"force_flush_interval"`
+	MaxDatumsPerCall         int             `mapstructure:"max_datums_per_call"`
+	MaxValuesPerDatum        int             `mapstructure:"max_values_per_datum"`
+	RollupDimensions         [][]string      `mapstructure:"rollup_dimensions,omitempty"`
+	DropOriginalConfigs      map[string]bool `mapstructure:"drop_original_metrics,omitempty"`
+	Namespace                string          `mapstructure:"namespace"`
 
 	// ResourceToTelemetrySettings is the option for converting resource
 	// attributes to telemetry attributes.

diff --git a/plugins/outputs/cloudwatch/config_test.go b/plugins/outputs/cloudwatch/config_test.go
@@ -104,12 +104,10 @@ func TestConfigDropOriginConfigs(t *testing.T) {
 	assert.Equal(t, 1, len(c.Exporters))
 	c2, ok := c.Exporters[component.NewID(TypeStr)].(*Config)
 	assert.True(t, ok)
-	drop := c2.DropOriginConfigs
+	drop := c2.DropOriginalConfigs
 	assert.NotEmpty(t, drop)
-	assert.Len(t, drop, 2)
-	assert.NotNil(t, drop["cpu"])
-	assert.Equal(t, []string{"time", "usage"}, drop["cpu"])
-	assert.NotNil(t, drop["foo"])
-	assert.Nil(t, drop["bar"])
-	assert.Equal(t, []string{"bar"}, drop["foo"])
+	assert.Len(t, drop, 3)
+	assert.True(t, drop["cpu_time"])
+	assert.True(t, drop["cpu_usage"])
+	assert.True(t, drop["foo_bar"])
 }
diff --git a/plugins/outputs/cloudwatch/testdata/drop_original.yaml b/plugins/outputs/cloudwatch/testdata/drop_original.yaml
@@ -5,11 +5,9 @@ exporters:
   awscloudwatch:
     region: us-yeast-99
     drop_original_metrics:
-      cpu:
-        - time
-        - usage
-      foo:
-        - bar
+      cpu_time: true
+      cpu_usage: true
+      foo_bar: true
 
 service:
   pipelines:

diff --git a/translator/tocwconfig/sampleConfig/complete_linux_config.yaml b/translator/tocwconfig/sampleConfig/complete_linux_config.yaml
@@ -2,9 +2,8 @@ connectors: {}
 exporters:
     awscloudwatch:
         drop_original_metrics:
-            cpu:
-                - cpu_usage_idle
-                - time_active
+            CPU_USAGE_IDLE: true
+            cpu_time_active: true
         endpoint_override: https://monitoring-fips.us-west-2.amazonaws.com
         force_flush_interval: 1m0s
         max_datums_per_call: 1000

diff --git a/translator/tocwconfig/sampleConfig/drop_origin_linux.yaml b/translator/tocwconfig/sampleConfig/drop_origin_linux.yaml
@@ -2,12 +2,10 @@ connectors: {}
 exporters:
     awscloudwatch:
         drop_original_metrics:
-            cpu:
-                - cpu_usage_idle
-                - time_active
-            nvidia_smi:
-                - utilization_gpu
-                - temperature_gpu
+            CPU_USAGE_IDLE: true
+            cpu_time_active: true
+            nvidia_smi_utilization_gpu: true
+            nvidia_smi_temperature_gpu: true
         force_flush_interval: 1m0s
         max_datums_per_call: 1000
         max_values_per_datum: 150

diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go
@@ -160,12 +160,6 @@ func TestAdvancedConfig(t *testing.T) {
 	checkTranslation(t, "advanced_config_windows", "windows", expectedEnvVars, "")
 }
 
-func TestDropOriginConfig(t *testing.T) {
-	resetContext(t)
-	expectedEnvVars := map[string]string{}
-	checkTranslation(t, "drop_origin_linux", "linux", expectedEnvVars, "")
-}
-
 func TestLogOnlyConfig(t *testing.T) {
 	resetContext(t)
 	expectedEnvVars := map[string]string{}

diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go
@@ -21,3 +21,9 @@ func TestCompleteConfigUnix(t *testing.T) {
 	checkTranslation(t, "complete_linux_config", "linux", expectedEnvVars, "")
 	checkTranslation(t, "complete_darwin_config", "darwin", nil, "")
 }
+
+func TestDropOriginConfig(t *testing.T) {
+	resetContext(t)
+	expectedEnvVars := map[string]string{}
+	checkTranslation(t, "drop_origin_linux", "linux", expectedEnvVars, "")
+}
diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go
@@ -38,6 +38,8 @@ const (
 	CredentialsKey                     = "credentials"
 	RoleARNKey                         = "role_arn"
 	MetricsCollectionIntervalKey       = "metrics_collection_interval"
+	MeasurementKey                     = "measurement"
+	DropOriginalMetricsKey             = "drop_original_metrics"
 	ForceFlushIntervalKey              = "force_flush_interval"
 	ContainerInsightsMetricGranularity = "metric_granularity"
 	PreferFullPodName                  = "prefer_full_pod_name"