redpanda: assert backwards compat guarantees

This commit utilizes `valuesutil.Generate` to assert the backwards
compatibility guarantees of the redpanda chart's values.schema.json
across all minor versions start with the 5.6.x series.

The guarantee is both documented and enforced by
`TestSchemaBackwardCompat` test.

Taskfile.yaml

@@ -82,6 +82,13 @@ tasks:
       - nix run .#genschema > charts/redpanda/values.schema.json
       # Generate helm templates from Go definitions
       - nix run .#gotohelm -- -write ./charts/redpanda/templates ./charts/redpanda 
+      # Copy the most recent values.schema.json of each minor version, starting
+      # with 5.6.x into testdata for backwards compat assertions.
+      # TODO should probably find a way to automate this as we'll need to add
+      # in a line for each minor version bump.
+      - |
+        git show $(git for-each-ref --sort=authordate --format '%(refname)' 'refs/tags/redpanda-5.7.*' | tail -n 1):charts/redpanda/values.schema.json > ./charts/redpanda/testdata/schemas/5.7.x.schema.json
+        git show $(git for-each-ref --sort=authordate --format '%(refname)' 'refs/tags/redpanda-5.6.*' | tail -n 1):charts/redpanda/values.schema.json > ./charts/redpanda/testdata/schemas/5.6.x.schema.json
 
   create-test-rack-awareness:
     cmds:

charts/redpanda/chart_test.go

@@ -2,18 +2,31 @@ package redpanda_test
 
 import (
 	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"math/rand"
 	"os"
 	"os/exec"
+	"path"
+	"reflect"
+	"slices"
 	"testing"
+	"testing/quick"
+	"time"
 
+	"github.com/invopop/jsonschema"
 	"github.com/redpanda-data/helm-charts/charts/redpanda"
 	"github.com/redpanda-data/helm-charts/pkg/helm"
 	"github.com/redpanda-data/helm-charts/pkg/helm/helmtest"
 	"github.com/redpanda-data/helm-charts/pkg/kube"
 	"github.com/redpanda-data/helm-charts/pkg/testutil"
+	"github.com/redpanda-data/helm-charts/pkg/valuesutil"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 )
 
 func TieredStorageStatic(t *testing.T) redpanda.PartialValues {
@@ -91,6 +104,96 @@ func TieredStorageSecretRefs(t *testing.T, secret *corev1.Secret) redpanda.Parti
 	}
 }
 
+// TestSchemaBackwardCompat asserts that all values.schema.json are backwards
+// compatible by one minor version **provided that no deprecated fields are
+// specified**.
+func TestSchemaBackwardCompat(t *testing.T) {
+	schemaFiles, err := os.ReadDir("testdata/schemas")
+	require.NoError(t, err)
+
+	var schemas []*jsonschema.Schema
+	var names []string
+
+	// TODO at somepoint we'll have to figure out how to sort these.
+	for _, schemaFile := range schemaFiles {
+		schemaBytes, err := os.ReadFile(path.Join("testdata/schemas", schemaFile.Name()))
+		require.NoError(t, err)
+
+		var data map[string]any
+		require.NoError(t, json.Unmarshal(schemaBytes, &data))
+
+		schema, err := valuesutil.UnmarshalInto[*jsonschema.Schema](fixSchema(data))
+		require.NoError(t, err)
+
+		schemas = append(schemas, schema)
+		names = append(names, schemaFile.Name()[:len(schemaFile.Name())-len(".schema.json")])
+	}
+
+	// Inject the most recent chart schema as HEAD to check for any breaking
+	// changes across patch versions or new minor versions.
+	schemas = append(schemas, redpanda.JSONSchema())
+	names = append(names, "HEAD")
+
+	for i := 0; i < len(names)-1; i++ {
+		fromName := names[i]
+		toName := names[i+1]
+		fromSchema := schemas[i]
+		toSchema := schemas[i+1]
+
+		t.Run(fmt.Sprintf("%s to %s", fromName, toName), func(t *testing.T) {
+			quick.Check(func(values map[string]any, schema *jsonschema.Schema) bool {
+				return valuesutil.Validate(schema, values) == nil
+			}, &quick.Config{
+				Values: func(v []reflect.Value, r *rand.Rand) {
+					// Generate a valid value from the previous schema.
+					v[0] = reflect.ValueOf(valuesutil.Generate(r, fromSchema))
+					// Validate it against the next schema.
+					v[1] = reflect.ValueOf(toSchema)
+				},
+			})
+		})
+	}
+}
+
+func TestTemplateProperties(t *testing.T) {
+	// - Setting statefulset.replicas > 1000 causes timeouts.
+	// - Not setting advertisedPorts causes out of index errors.
+	t.Skip("Currently finding too many failures")
+
+	ctx := testutil.Context(t)
+	client, err := helm.New(helm.Options{ConfigHome: testutil.TempDir(t)})
+	require.NoError(t, err)
+
+	f := func(values *redpanda.PartialValues) error {
+		// Helm template can hang on some values. We don't want that to stall
+		// tests nor would we want customers to experience that. Cut it off
+		// with a deadline.
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+
+		_, err := client.Template(ctx, ".", helm.TemplateOptions{
+			Name:   "redpanda",
+			Values: values,
+			Set: []string{
+				"tests.enabled=false",
+			},
+		})
+		return err
+	}
+
+	err = quick.Check(func(values *redpanda.PartialValues) bool {
+		return f(values) == nil
+	}, &quick.Config{
+		Values: func(args []reflect.Value, rng *rand.Rand) {
+			values := valuesutil.Generate(rng, redpanda.JSONSchema())
+			partial, _ := valuesutil.UnmarshalInto[redpanda.PartialValues](values)
+			FixPartialCerts(&partial)
+			args[0] = reflect.ValueOf(&partial)
+		},
+	})
+	require.NoError(t, err)
+}
+
 func TestTemplate(t *testing.T) {
 	ctx := testutil.Context(t)
 	client, err := helm.New(helm.Options{ConfigHome: testutil.TempDir(t)})
@@ -142,7 +245,7 @@ func TestTemplate(t *testing.T) {
 				t.Logf("kube-linter error(s) found for %q: \n%s", v.Name(), errKubeLinter)
 			}
 			// TODO: remove comment below and the logging above once we agree to linter
-			//require.NoError(t, errKubeLinter)
+			// require.NoError(t, errKubeLinter)
 
 			testutil.AssertGolden(t, testutil.YAML, "./testdata/"+v.Name()+".golden", out)
 		})
@@ -195,3 +298,95 @@ func TestChart(t *testing.T) {
 		require.Equal(t, "from-secret-access-key", config["cloud_storage_access_key"])
 	})
 }
+
+// FixPartialCerts is a helper for tests utilizing valuesutil.Generate. There's
+// no way to constraint values in a JSON schema to those that exist as keys of
+// another object (our TLS certs). FixPartialCerts will traverse a
+// [redpanda.PartialValues] and retroactively create certificates if they don't
+// exist to ensure validity.
+func FixPartialCerts(values *redpanda.PartialValues) {
+	if values.TLS == nil {
+		values.TLS = &redpanda.PartialTLS{Enabled: ptr.To(true)}
+	}
+
+	if values.TLS.Certs == nil {
+		values.TLS.Certs = &redpanda.PartialTLSCertMap{}
+	}
+
+	expectedCerts := []*redpanda.PartialExternalTLS{
+		values.Listeners.Admin.TLS,
+		values.Listeners.Kafka.TLS,
+		values.Listeners.HTTP.TLS,
+	}
+
+	for _, cert := range expectedCerts {
+		if cert == nil {
+			continue
+		}
+
+		if _, ok := (*values.TLS.Certs)[*cert.Cert]; ok {
+			continue
+		}
+
+		(*values.TLS.Certs)[*cert.Cert] = redpanda.PartialTLSCert{CAEnabled: ptr.To(false)}
+	}
+}
+
+// fixSchema fixes minor issues with older hand written jsonschemas and expands
+// arrays in "type" to oneOf's as our jsonschema library requires type to be a
+// string.
+// NB: Dynamic fixing was elected to ensure that it's easy to repopulate
+// testdata/schema with raw git commands.
+func fixSchema(schema map[string]any) map[string]any {
+	for _, propKey := range []string{"properties", "patternProperties", "additionalProperties"} {
+		if props, ok := schema[propKey].(map[string]any); ok {
+			for key, value := range props {
+				if asMap, ok := value.(map[string]any); ok {
+					props[key] = fixSchema(asMap)
+				}
+			}
+		}
+	}
+
+	if items, ok := schema["items"].(map[string]any); ok {
+		schema["items"] = fixSchema(items)
+	}
+
+	if typeArr, ok := schema["type"].([]any); ok {
+		var oneOf []map[string]any
+		for _, t := range typeArr {
+			clone := maps.Clone(schema)
+			clone["type"] = t
+			oneOf = append(oneOf, clone)
+		}
+		schema = map[string]any{"oneOf": oneOf}
+	}
+
+	// Rename parameters to properties.
+	if props, ok := schema["parameters"]; ok {
+		schema["properties"] = props
+		delete(schema, "parameters")
+	}
+
+	// Add missing object types.
+	_, hasType := schema["type"]
+	_, hasProps := schema["properties"]
+	if !hasType && hasProps {
+		schema["type"] = "object"
+	}
+
+	// Remove any unspecified properties from required.
+	if required, ok := schema["required"].([]any); ok {
+		for i := 0; i < len(required); i++ {
+			key := required[i].(string)
+			_, hasProp := schema["properties"].(map[string]any)[key]
+			if !hasProp {
+				required = slices.Delete(required, i, i+1)
+				i--
+			}
+		}
+		schema["required"] = required
+	}
+
+	return schema
+}

charts/redpanda/schema.go

@@ -0,0 +1,20 @@
+// +gotohelm:ignore=true
+package redpanda
+
+import (
+	_ "embed"
+	"encoding/json"
+
+	"github.com/invopop/jsonschema"
+)
+
+//go:embed values.schema.json
+var schemaBytes []byte
+
+func JSONSchema() *jsonschema.Schema {
+	var s jsonschema.Schema
+	if err := json.Unmarshal(schemaBytes, &s); err != nil {
+		panic(err)
+	}
+	return &s
+}