✨ metrics: add generator for kube-state-metrics customresource configuration

cmd/controller-gen/main.go

@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/controller-tools/pkg/genall/help"
 	prettyhelp "sigs.k8s.io/controller-tools/pkg/genall/help/pretty"
 	"sigs.k8s.io/controller-tools/pkg/markers"
+	"sigs.k8s.io/controller-tools/pkg/metrics"
 	"sigs.k8s.io/controller-tools/pkg/rbac"
 	"sigs.k8s.io/controller-tools/pkg/schemapatcher"
 	"sigs.k8s.io/controller-tools/pkg/version"
@@ -53,6 +54,7 @@ var (
 		"object":      deepcopy.Generator{},
 		"webhook":     webhook.Generator{},
 		"schemapatch": schemapatcher.Generator{},
+		"metrics":     metrics.Generator{},
 	}
 
 	// allOutputRules defines the list of all known output rules, giving

go.mod

@@ -16,6 +16,7 @@ require (
 	k8s.io/api v0.31.1
 	k8s.io/apiextensions-apiserver v0.31.1
 	k8s.io/apimachinery v0.31.1
+	k8s.io/client-go v0.31.1
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/yaml v1.4.0
 )

go.sum

@@ -177,6 +177,8 @@ k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/
 k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ=
 k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U=
 k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0=
+k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=

pkg/metrics/generate_integration_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2023 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"sigs.k8s.io/controller-tools/pkg/genall"
+	"sigs.k8s.io/controller-tools/pkg/loader"
+	"sigs.k8s.io/controller-tools/pkg/markers"
+	"sigs.k8s.io/controller-tools/pkg/metrics/internal/config"
+)
+
+func Test_Generate(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Error(err)
+	}
+
+	optionsRegistry := &markers.Registry{}
+
+	metricGenerator := Generator{}
+	if err := metricGenerator.RegisterMarkers(optionsRegistry); err != nil {
+		t.Error(err)
+	}
+
+	out := &outputRule{
+		buf: &bytes.Buffer{},
+	}
+
+	// Load the passed packages as roots.
+	roots, err := loader.LoadRoots(path.Join(cwd, "testdata", "..."))
+	if err != nil {
+		t.Errorf("loading packages %v", err)
+	}
+
+	gen := Generator{}
+
+	generationContext := &genall.GenerationContext{
+		Collector:  &markers.Collector{Registry: optionsRegistry},
+		Roots:      roots,
+		Checker:    &loader.TypeChecker{},
+		OutputRule: out,
+	}
+
+	t.Log("Trying to generate a custom resource configuration from the loaded packages")
+
+	if err := gen.Generate(generationContext); err != nil {
+		t.Error(err)
+	}
+
+	output := strings.Split(out.buf.String(), "\n---\n")
+
+	header := fmt.Sprintf(headerText, "(devel)", config.KubeStateMetricsVersion)
+
+	if len(output) != 3 {
+		t.Error("Expected two output files, metrics configuration followed by rbac.")
+		return
+	}
+
+	generatedData := map[string]string{
+		"metrics.yaml": header + "---\n" + string(output[1]),
+		"rbac.yaml":    "---\n" + string(output[2]),
+	}
+
+	t.Log("Comparing output to testdata to check for regressions")
+
+	for _, golden := range []string{"metrics.yaml", "rbac.yaml"} {
+		// generatedRaw := strings.TrimSpace(output[i])
+
+		expectedRaw, err := os.ReadFile(path.Clean(path.Join(cwd, "testdata", golden)))
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		// Remove leading `---` and trim newlines
+		generated := strings.TrimSpace(strings.TrimPrefix(generatedData[golden], "---"))
+		expected := strings.TrimSpace(strings.TrimPrefix(string(expectedRaw), "---"))
+
+		diff := cmp.Diff(expected, generated)
+		if diff != "" {
+			t.Log("generated:")
+			t.Log(generated)
+			t.Log("diff:")
+			t.Log(diff)
+			t.Logf("Expected output to match file `testdata/%s` but it does not.", golden)
+			t.Logf("If the change is intended, use `go generate ./pkg/metrics/testdata` to regenerate the `testdata/%s` file.", golden)
+			t.Errorf("Detected a diff between the output of the integration test and the file `testdata/%s`.", golden)
+			return
+		}
+	}
+}
+
+type outputRule struct {
+	buf *bytes.Buffer
+}
+
+func (o *outputRule) Open(_ *loader.Package, _ string) (io.WriteCloser, error) {
+	return nopCloser{o.buf}, nil
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+func (n nopCloser) Close() error {
+	return nil
+}

pkg/metrics/generator.go

@@ -0,0 +1,183 @@
+/*
+Copyright 2024 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package metrics contain libraries for generating custom resource metrics configurations
+// for kube-state-metrics from metrics markers in Go source files.
+package metrics
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/gobuffalo/flect"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"sigs.k8s.io/controller-tools/pkg/crd"
+	"sigs.k8s.io/controller-tools/pkg/genall"
+	"sigs.k8s.io/controller-tools/pkg/loader"
+	ctrlmarkers "sigs.k8s.io/controller-tools/pkg/markers"
+	"sigs.k8s.io/controller-tools/pkg/metrics/internal/config"
+	"sigs.k8s.io/controller-tools/pkg/metrics/markers"
+	"sigs.k8s.io/controller-tools/pkg/rbac"
+	"sigs.k8s.io/controller-tools/pkg/version"
+)
+
+// Generator generates kube-state-metrics custom resource configuration files.
+type Generator struct{}
+
+var _ genall.Generator = &Generator{}
+var _ genall.NeedsTypeChecking = &Generator{}
+
+// RegisterMarkers registers all markers needed by this Generator
+// into the given registry.
+func (g Generator) RegisterMarkers(into *ctrlmarkers.Registry) error {
+	for _, m := range markers.MarkerDefinitions {
+		if err := m.Register(into); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+const headerText = `# Generated by controller-gen version %s
+# Generated based on types for kube-state-metrics %s
+`
+
+// Generate generates artifacts produced by this marker.
+// It's called after RegisterMarkers has been called.
+func (g Generator) Generate(ctx *genall.GenerationContext) error {
+	// Create the parser which is specific to the metric generator.
+	parser := newParser(
+		&crd.Parser{
+			Collector: ctx.Collector,
+			Checker:   ctx.Checker,
+		},
+	)
+
+	// Loop over all passed packages.
+	for _, pkg := range ctx.Roots {
+		// skip packages which don't import metav1 because they can't define a CRD without meta v1.
+		metav1 := pkg.Imports()["k8s.io/apimachinery/pkg/apis/meta/v1"]
+		if metav1 == nil {
+			continue
+		}
+
+		// parse the given package to feed crd.FindKubeKinds with Kubernetes Objects.
+		parser.NeedPackage(pkg)
+
+		kubeKinds := crd.FindKubeKinds(parser.Parser, metav1)
+		if len(kubeKinds) == 0 {
+			// no objects in the roots
+			return nil
+		}
+
+		// Create metrics for all Custom Resources in this package.
+		// This creates the customresourcestate.Resource object which contains all metric
+		// definitions for the Custom Resource, if it is part of the package.
+		for _, gv := range kubeKinds {
+			if err := parser.NeedResourceFor(pkg, gv); err != nil {
+				return err
+			}
+		}
+	}
+
+	// Initialize empty customresourcestate configuration file and fill it with the
+	// customresourcestate.Resource objects from the parser.
+	metrics := config.Metrics{
+		Spec: config.MetricsSpec{
+			Resources: []config.Resource{},
+		},
+	}
+
+	rules := []*rbac.Rule{}
+
+	for _, resource := range parser.CustomResourceStates {
+		if resource == nil {
+			continue
+		}
+		if len(resource.Metrics) > 0 {
+			// Sort the metrics to get a deterministic output.
+			sort.Slice(resource.Metrics, func(i, j int) bool {
+				return resource.Metrics[i].Name < resource.Metrics[j].Name
+			})
+
+			metrics.Spec.Resources = append(metrics.Spec.Resources, *resource)
+
+			rules = append(rules, &rbac.Rule{
+				Groups:    []string{resource.GroupVersionKind.Group},
+				Resources: []string{strings.ToLower(flect.Pluralize(resource.GroupVersionKind.Kind))},
+				Verbs:     []string{"get", "list", "watch"},
+			})
+		}
+	}
+
+	// Sort the resources by GVK to get a deterministic output.
+	sort.Slice(metrics.Spec.Resources, func(i, j int) bool {
+		a := metrics.Spec.Resources[i].GroupVersionKind.String()
+		b := metrics.Spec.Resources[j].GroupVersionKind.String()
+		return a < b
+	})
+
+	header := fmt.Sprintf(headerText, version.Version(), config.KubeStateMetricsVersion)
+
+	// Write the rendered yaml to the context which will result in stdout.
+	virtualFilePath := "metrics.yaml"
+	if err := ctx.WriteYAML(virtualFilePath, header, []interface{}{metrics}, genall.WithTransform(addCustomResourceStateKind)); err != nil {
+		return fmt.Errorf("WriteYAML to %s: %w", virtualFilePath, err)
+	}
+
+	clusterRole := rbacv1.ClusterRole{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ClusterRole",
+			APIVersion: rbacv1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "manager-metrics-role",
+			Labels: map[string]string{
+				"kube-state-metrics/aggregate-to-manager": "true",
+			},
+		},
+		Rules: rbac.NormalizeRules(rules),
+	}
+
+	virtualFilePath = "rbac.yaml"
+	if err := ctx.WriteYAML(virtualFilePath, "", []interface{}{clusterRole}, genall.WithTransform(genall.TransformRemoveCreationTimestamp)); err != nil {
+		return fmt.Errorf("WriteYAML to %s: %w", virtualFilePath, err)
+	}
+
+	return nil
+}
+
+// CheckFilter indicates the loader.NodeFilter (if any) that should be used
+// to prune out unused types/packages when type-checking (nodes for which
+// the filter returns true are considered "interesting").  This filter acts
+// as a baseline -- all types the pass through this filter will be checked,
+// but more than that may also be checked due to other generators' filters.
+func (Generator) CheckFilter() loader.NodeFilter {
+	// Re-use controller-tools filter to filter out unrelated nodes that aren't used
+	// in CRD generation, like interfaces and struct fields without JSON tag.
+	return crd.Generator{}.CheckFilter()
+}
+
+// addCustomResourceStateKind adds the correct kind because we don't have a correct
+// kubernetes-style object as configuration definition.
+func addCustomResourceStateKind(obj map[string]interface{}) error {
+	obj["kind"] = "CustomResourceStateMetrics"
+	return nil
+}