diff --git a/CHANGELOG.md b/CHANGELOG.md index 280fcfc171..0a43946841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ ### [Unreleased](https://github.com/redpanda-data/helm-charts/releases/tag/redpanda-FILLMEIN) - YYYY-MM-DD #### Added +* Added `resources.limits` and `resources.requests` as an alternative method of managing the redpanda container's resources. + + When both `resources.limits` and `resources.requests` are specified, the + redpanda container's `resources` will be set to the provided values and all + other keys of `resources` will be ignored. Instead, all other values will be + inferred from the limits and requests. + + This allows fine grain control of resources. i.e. It is now possible to set + CPU requests without setting limits: + + ```yaml + resources: + limits: {} # Specified but not cpu or memory values provided + requests: + cpu: 5 # Only CPU requests + ``` + + For more details see [redpanda's values.yaml](./charts/redpanda/values.yaml). + #### Changed #### Fixed #### Removed diff --git a/charts/redpanda/README.md b/charts/redpanda/README.md index 380df13d21..c86e96948f 100644 --- a/charts/redpanda/README.md +++ b/charts/redpanda/README.md @@ -621,7 +621,7 @@ Enable for features that need extra privileges. If you use the Redpanda Operator ### [resources](https://artifacthub.io/packages/helm/redpanda-data/redpanda?modal=values&path=resources) -Pod resource management. This section simplifies resource allocation by providing a single location where resources are defined. Helm sets these resource values within the `statefulset.yaml` and `configmap.yaml` templates. The default values are for a development environment. Production-level values and other considerations are documented, where those values are different from the default. For details, see the [Pod resources documentation](https://docs.redpanda.com/docs/manage/kubernetes/manage-resources/). +Pod resource management. This section simplifies resource allocation for the redpanda container by providing a single location where resources are defined. Resources may be specified by either setting `resources.cpu` and `resources.memory` (the default) or by setting `resources.requests` and `resources.limits`. For details on `resources.cpu` and `resources.memory`, see their respective documentation below. When `resources.limits` and `resources.requests` are set, the redpanda container's resources will be set to exactly the provided values. This allows users to granularly control limits and requests to best suite their use case. For example: `resources.requests.cpu` may be set without setting `resources.limits.cpu` to avoid the potential of CPU throttling. Redpanda's resource related CLI flags will then be calculated as follows: * `--smp floor(resources.{requests,limits}.cpu)` * `--memory resources.{requests,limits}.memory * 90%` * `--reserve-memory 0` * `--overprovisioned resources.{requests,limits}.cpu < 1000m` If neither a request nor a limit is for cpu or memory provided, the corresponding flag will be omitted. i.e. Setting `resources.limits` and `resources.requests` to `{}` will result in redpanda being run without `--smp` or `--memory`. (This is not recommended). If the computed CLI flags are undesirable, they may be overridden by specifying the desired value through `statefulset.additionalRedpandaCmdFlags`. The default values are for a development environment. Production-level values and other considerations are documented, where those values are different from the default. For details, see the [Pod resources documentation](https://docs.redpanda.com/docs/manage/kubernetes/manage-resources/). **Default:** diff --git a/charts/redpanda/configmap.tpl.go b/charts/redpanda/configmap.tpl.go index 6229cfbbea..3c50d1d82c 100644 --- a/charts/redpanda/configmap.tpl.go +++ b/charts/redpanda/configmap.tpl.go @@ -365,8 +365,6 @@ func rpkNodeConfig(dot *helmette.Dot) map[string]any { } result := map[string]any{ - "overprovisioned": values.Resources.GetOverProvisionValue(), - "enable_memory_locking": ptr.Deref(values.Resources.Memory.EnableMemoryLocking, false), "additional_start_flags": RedpandaAdditionalStartFlags(dot), "kafka_api": map[string]any{ "brokers": brokerList, @@ -622,7 +620,7 @@ func RedpandaAdditionalStartFlags(dot *helmette.Dot) []string { values := helmette.Unwrap[Values](dot.Values) // All `additional_start_flags` that are set by the chart. - chartFlags := values.Resources.GetRedpandaStartFlags() + chartFlags := values.Resources.GetRedpandaFlags() chartFlags["default-log-level"] = values.Logging.LogLevel // If in developer_mode, don't set reserve-memory. @@ -646,7 +644,14 @@ func RedpandaAdditionalStartFlags(dot *helmette.Dot) []string { flags := []string{} for _, key := range keys { - flags = append(flags, fmt.Sprintf("--%s=%s", key, chartFlags[key])) + value := chartFlags[key] + // Support flags that don't have values (`--overprovisioned`) by + // letting them be specified as key: "" + if value == "" { + flags = append(flags, fmt.Sprintf("--%s", key)) + } else { + flags = append(flags, fmt.Sprintf("--%s=%s", key, value)) + } } return append(flags, values.Statefulset.AdditionalRedpandaCmdFlags...) diff --git a/charts/redpanda/templates/_configmap.go.tpl b/charts/redpanda/templates/_configmap.go.tpl index 478919586d..7f96dab54d 100644 --- a/charts/redpanda/templates/_configmap.go.tpl +++ b/charts/redpanda/templates/_configmap.go.tpl @@ -313,7 +313,7 @@ {{- if (gt ((get (fromJson (include "_shims.len" (dict "a" (list $tls_8) ))) "r") | int) (0 | int)) -}} {{- $schemaRegistryTLS = $tls_8 -}} {{- end -}} -{{- $result := (dict "overprovisioned" (get (fromJson (include "redpanda.RedpandaResources.GetOverProvisionValue" (dict "a" (list $values.resources) ))) "r") "enable_memory_locking" (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $values.resources.memory.enable_memory_locking false) ))) "r") "additional_start_flags" (get (fromJson (include "redpanda.RedpandaAdditionalStartFlags" (dict "a" (list $dot) ))) "r") "kafka_api" (dict "brokers" $brokerList "tls" $brokerTLS ) "admin_api" (dict "addresses" (get (fromJson (include "redpanda.Listeners.AdminList" (dict "a" (list $values.listeners ($values.statefulset.replicas | int) (get (fromJson (include "redpanda.Fullname" (dict "a" (list $dot) ))) "r") (get (fromJson (include "redpanda.InternalDomain" (dict "a" (list $dot) ))) "r")) ))) "r") "tls" $adminTLS ) "schema_registry" (dict "addresses" (get (fromJson (include "redpanda.Listeners.SchemaRegistryList" (dict "a" (list $values.listeners ($values.statefulset.replicas | int) (get (fromJson (include "redpanda.Fullname" (dict "a" (list $dot) ))) "r") (get (fromJson (include "redpanda.InternalDomain" (dict "a" (list $dot) ))) "r")) ))) "r") "tls" $schemaRegistryTLS ) ) -}} +{{- $result := (dict "additional_start_flags" (get (fromJson (include "redpanda.RedpandaAdditionalStartFlags" (dict "a" (list $dot) ))) "r") "kafka_api" (dict "brokers" $brokerList "tls" $brokerTLS ) "admin_api" (dict "addresses" (get (fromJson (include "redpanda.Listeners.AdminList" (dict "a" (list $values.listeners ($values.statefulset.replicas | int) (get (fromJson (include "redpanda.Fullname" (dict "a" (list $dot) ))) "r") (get (fromJson (include "redpanda.InternalDomain" (dict "a" (list $dot) ))) "r")) ))) "r") "tls" $adminTLS ) "schema_registry" (dict "addresses" (get (fromJson (include "redpanda.Listeners.SchemaRegistryList" (dict "a" (list $values.listeners ($values.statefulset.replicas | int) (get (fromJson (include "redpanda.Fullname" (dict "a" (list $dot) ))) "r") (get (fromJson (include "redpanda.InternalDomain" (dict "a" (list $dot) ))) "r")) ))) "r") "tls" $schemaRegistryTLS ) ) -}} {{- $result = (merge (dict ) $result (get (fromJson (include "redpanda.Tuning.Translate" (dict "a" (list $values.tuning) ))) "r")) -}} {{- $result = (merge (dict ) $result (get (fromJson (include "redpanda.Config.CreateRPKConfiguration" (dict "a" (list $values.config) ))) "r")) -}} {{- $_is_returning = true -}} @@ -544,7 +544,7 @@ {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} {{- $values := $dot.Values.AsMap -}} -{{- $chartFlags := (get (fromJson (include "redpanda.RedpandaResources.GetRedpandaStartFlags" (dict "a" (list $values.resources) ))) "r") -}} +{{- $chartFlags := (get (fromJson (include "redpanda.RedpandaResources.GetRedpandaFlags" (dict "a" (list $values.resources) ))) "r") -}} {{- $_ := (set $chartFlags "default-log-level" $values.logging.logLevel) -}} {{- if (eq (index $values.config.node "developer_mode") true) -}} {{- $_ := (unset $chartFlags "reserve-memory") -}} @@ -566,7 +566,12 @@ {{- $_ := (sortAlpha $keys) -}} {{- $flags := (list ) -}} {{- range $_, $key := $keys -}} -{{- $flags = (concat (default (list ) $flags) (list (printf "--%s=%s" $key (ternary (index $chartFlags $key) "" (hasKey $chartFlags $key))))) -}} +{{- $value := (ternary (index $chartFlags $key) "" (hasKey $chartFlags $key)) -}} +{{- if (eq $value "") -}} +{{- $flags = (concat (default (list ) $flags) (list (printf "--%s" $key))) -}} +{{- else -}} +{{- $flags = (concat (default (list ) $flags) (list (printf "--%s=%s" $key $value))) -}} +{{- end -}} {{- end -}} {{- if $_is_returning -}} {{- break -}} diff --git a/charts/redpanda/templates/_values.go.tpl b/charts/redpanda/templates/_values.go.tpl index 309dbda123..f3dacf9de7 100644 --- a/charts/redpanda/templates/_values.go.tpl +++ b/charts/redpanda/templates/_values.go.tpl @@ -106,6 +106,11 @@ {{- $rr := (index .a 0) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} +{{- if (and (ne (toJson $rr.limits) "null") (ne (toJson $rr.requests) "null")) -}} +{{- $_is_returning = true -}} +{{- (dict "r" (mustMergeOverwrite (dict ) (dict "limits" $rr.limits "requests" $rr.requests ))) | toJson -}} +{{- break -}} +{{- end -}} {{- $reqs := (mustMergeOverwrite (dict ) (dict "limits" (dict "cpu" $rr.cpu.cores "memory" $rr.memory.container.max ) )) -}} {{- if (ne (toJson $rr.memory.container.min) "null") -}} {{- $_ := (set $reqs "requests" (dict "cpu" $rr.cpu.cores "memory" $rr.memory.container.min )) -}} @@ -116,19 +121,25 @@ {{- end -}} {{- end -}} -{{- define "redpanda.RedpandaResources.GetRedpandaStartFlags" -}} +{{- define "redpanda.RedpandaResources.GetRedpandaFlags" -}} {{- $rr := (index .a 0) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} -{{- $flags := (dict ) -}} -{{- $coresInMillies_2 := ((get (fromJson (include "_shims.resource_MilliValue" (dict "a" (list $rr.cpu.cores) ))) "r") | int64) -}} -{{- if (lt $coresInMillies_2 (1000 | int64)) -}} -{{- $_ := (set $flags "smp" (printf "%d" (1 | int))) -}} -{{- else -}} -{{- $_ := (set $flags "smp" (printf "%d" ((get (fromJson (include "_shims.resource_Value" (dict "a" (list $rr.cpu.cores) ))) "r") | int64))) -}} +{{- $flags := (dict "reserve-memory" (printf "%dM" ((get (fromJson (include "redpanda.RedpandaResources.reserveMemory" (dict "a" (list $rr) ))) "r") | int64)) ) -}} +{{- $smp_2 := (get (fromJson (include "redpanda.RedpandaResources.smp" (dict "a" (list $rr) ))) "r") -}} +{{- if (ne (toJson $smp_2) "null") -}} +{{- $_ := (set $flags "smp" (printf "%d" ($smp_2 | int64))) -}} +{{- end -}} +{{- $memory_3 := (get (fromJson (include "redpanda.RedpandaResources.memory" (dict "a" (list $rr) ))) "r") -}} +{{- if (ne (toJson $memory_3) "null") -}} +{{- $_ := (set $flags "memory" (printf "%dM" ($memory_3 | int64))) -}} +{{- end -}} +{{- if (and (eq (toJson $rr.limits) "null") (eq (toJson $rr.requests) "null")) -}} +{{- $_ := (set $flags "lock-memory" (printf "%v" (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $rr.memory.enable_memory_locking false) ))) "r"))) -}} +{{- end -}} +{{- if (get (fromJson (include "redpanda.RedpandaResources.GetOverProvisionValue" (dict "a" (list $rr) ))) "r") -}} +{{- $_ := (set $flags "overprovisioned" "") -}} {{- end -}} -{{- $_ := (set $flags "memory" (printf "%dM" ((get (fromJson (include "redpanda.RedpandaResources.memory" (dict "a" (list $rr) ))) "r") | int64))) -}} -{{- $_ := (set $flags "reserve-memory" (printf "%dM" ((get (fromJson (include "redpanda.RedpandaResources.reserveMemory" (dict "a" (list $rr) ))) "r") | int64))) -}} {{- $_is_returning = true -}} {{- (dict "r" $flags) | toJson -}} {{- break -}} @@ -139,6 +150,24 @@ {{- $rr := (index .a 0) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} +{{- if (and (ne (toJson $rr.limits) "null") (ne (toJson $rr.requests) "null")) -}} +{{- $_439_cpuReq_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list ($rr.requests) "cpu" "0") ))) "r") -}} +{{- $cpuReq := (index $_439_cpuReq_ok 0) -}} +{{- $ok := (index $_439_cpuReq_ok 1) -}} +{{- if (not $ok) -}} +{{- $_441_cpuReq_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list ($rr.limits) "cpu" "0") ))) "r") -}} +{{- $cpuReq = (index $_441_cpuReq_ok 0) -}} +{{- $ok = (index $_441_cpuReq_ok 1) -}} +{{- end -}} +{{- if (and $ok (lt ((get (fromJson (include "_shims.resource_MilliValue" (dict "a" (list $cpuReq) ))) "r") | int64) (1000 | int64))) -}} +{{- $_is_returning = true -}} +{{- (dict "r" true) | toJson -}} +{{- break -}} +{{- end -}} +{{- $_is_returning = true -}} +{{- (dict "r" false) | toJson -}} +{{- break -}} +{{- end -}} {{- if (lt ((get (fromJson (include "_shims.resource_MilliValue" (dict "a" (list $rr.cpu.cores) ))) "r") | int64) (1000 | int64)) -}} {{- $_is_returning = true -}} {{- (dict "r" true) | toJson -}} @@ -150,15 +179,72 @@ {{- end -}} {{- end -}} +{{- define "redpanda.RedpandaResources.smp" -}} +{{- $rr := (index .a 0) -}} +{{- range $_ := (list 1) -}} +{{- $_is_returning := false -}} +{{- if (and (ne (toJson $rr.limits) "null") (ne (toJson $rr.requests) "null")) -}} +{{- $_463_cpuReq_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list ($rr.requests) "cpu" "0") ))) "r") -}} +{{- $cpuReq := (index $_463_cpuReq_ok 0) -}} +{{- $ok := (index $_463_cpuReq_ok 1) -}} +{{- if (not $ok) -}} +{{- $_465_cpuReq_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list ($rr.limits) "cpu" "0") ))) "r") -}} +{{- $cpuReq = (index $_465_cpuReq_ok 0) -}} +{{- $ok = (index $_465_cpuReq_ok 1) -}} +{{- end -}} +{{- if (not $ok) -}} +{{- $_is_returning = true -}} +{{- (dict "r" (coalesce nil)) | toJson -}} +{{- break -}} +{{- end -}} +{{- $smp := ((div ((get (fromJson (include "_shims.resource_MilliValue" (dict "a" (list $cpuReq) ))) "r") | int64) (1000 | int64)) | int64) -}} +{{- if (lt $smp (1 | int64)) -}} +{{- $smp = (1 | int64) -}} +{{- end -}} +{{- $_is_returning = true -}} +{{- (dict "r" $smp) | toJson -}} +{{- break -}} +{{- end -}} +{{- $coresInMillies_4 := ((get (fromJson (include "_shims.resource_MilliValue" (dict "a" (list $rr.cpu.cores) ))) "r") | int64) -}} +{{- if (lt $coresInMillies_4 (1000 | int64)) -}} +{{- $_is_returning = true -}} +{{- (dict "r" ((1 | int64) | int64)) | toJson -}} +{{- break -}} +{{- end -}} +{{- $_is_returning = true -}} +{{- (dict "r" (((get (fromJson (include "_shims.resource_Value" (dict "a" (list $rr.cpu.cores) ))) "r") | int64) | int64)) | toJson -}} +{{- break -}} +{{- end -}} +{{- end -}} + {{- define "redpanda.RedpandaResources.memory" -}} {{- $rr := (index .a 0) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} +{{- if (and (ne (toJson $rr.limits) "null") (ne (toJson $rr.requests) "null")) -}} +{{- $_520_memReq_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list ($rr.requests) "memory" "0") ))) "r") -}} +{{- $memReq := (index $_520_memReq_ok 0) -}} +{{- $ok := (index $_520_memReq_ok 1) -}} +{{- if (not $ok) -}} +{{- $_522_memReq_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list ($rr.limits) "memory" "0") ))) "r") -}} +{{- $memReq = (index $_522_memReq_ok 0) -}} +{{- $ok = (index $_522_memReq_ok 1) -}} +{{- end -}} +{{- if (not $ok) -}} +{{- $_is_returning = true -}} +{{- (dict "r" (coalesce nil)) | toJson -}} +{{- break -}} +{{- end -}} +{{- $memory := (((mulf (((get (fromJson (include "_shims.resource_Value" (dict "a" (list $memReq) ))) "r") | int64) | float64) 0.90) | float64) | int64) -}} +{{- $_is_returning = true -}} +{{- (dict "r" ((div $memory ((mul (1024 | int) (1024 | int)))) | int64)) | toJson -}} +{{- break -}} +{{- end -}} {{- $memory := ((0 | int64) | int64) -}} {{- $containerMemory := ((get (fromJson (include "redpanda.RedpandaResources.containerMemory" (dict "a" (list $rr) ))) "r") | int64) -}} -{{- $rpMem_3 := $rr.memory.redpanda -}} -{{- if (and (ne (toJson $rpMem_3) "null") (ne (toJson $rpMem_3.memory) "null")) -}} -{{- $memory = ((div ((get (fromJson (include "_shims.resource_Value" (dict "a" (list $rpMem_3.memory) ))) "r") | int64) ((mul (1024 | int) (1024 | int)))) | int64) -}} +{{- $rpMem_5 := $rr.memory.redpanda -}} +{{- if (and (ne (toJson $rpMem_5) "null") (ne (toJson $rpMem_5.memory) "null")) -}} +{{- $memory = ((div ((get (fromJson (include "_shims.resource_Value" (dict "a" (list $rpMem_5.memory) ))) "r") | int64) ((mul (1024 | int) (1024 | int)))) | int64) -}} {{- else -}} {{- $memory = (((mulf ($containerMemory | float64) 0.8) | float64) | int64) -}} {{- end -}} @@ -168,7 +254,7 @@ {{- if (lt $memory (256 | int64)) -}} {{- $_ := (fail (printf "%d is below the minimum value for Redpanda" $memory)) -}} {{- end -}} -{{- if (gt ((add $memory ((get (fromJson (include "redpanda.RedpandaResources.reserveMemory" (dict "a" (list $rr) ))) "r") | int64)) | int64) $containerMemory) -}} +{{- if (gt ((add $memory (((get (fromJson (include "redpanda.RedpandaResources.reserveMemory" (dict "a" (list $rr) ))) "r") | int64) | int64)) | int64) $containerMemory) -}} {{- $_ := (fail (printf "Not enough container memory for Redpanda memory values where Redpanda: %d, reserve: %d, container: %d" $memory ((get (fromJson (include "redpanda.RedpandaResources.reserveMemory" (dict "a" (list $rr) ))) "r") | int64) $containerMemory)) -}} {{- end -}} {{- $_is_returning = true -}} @@ -181,10 +267,15 @@ {{- $rr := (index .a 0) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} -{{- $rpMem_4 := $rr.memory.redpanda -}} -{{- if (and (ne (toJson $rpMem_4) "null") (ne (toJson $rpMem_4.reserveMemory) "null")) -}} +{{- if (and (ne (toJson $rr.limits) "null") (ne (toJson $rr.requests) "null")) -}} +{{- $_is_returning = true -}} +{{- (dict "r" (0 | int64)) | toJson -}} +{{- break -}} +{{- end -}} +{{- $rpMem_6 := $rr.memory.redpanda -}} +{{- if (and (ne (toJson $rpMem_6) "null") (ne (toJson $rpMem_6.reserveMemory) "null")) -}} {{- $_is_returning = true -}} -{{- (dict "r" ((div ((get (fromJson (include "_shims.resource_Value" (dict "a" (list $rpMem_4.reserveMemory) ))) "r") | int64) ((mul (1024 | int) (1024 | int)))) | int64)) | toJson -}} +{{- (dict "r" ((div ((get (fromJson (include "_shims.resource_Value" (dict "a" (list $rpMem_6.reserveMemory) ))) "r") | int64) ((mul (1024 | int) (1024 | int)))) | int64)) | toJson -}} {{- break -}} {{- end -}} {{- $_is_returning = true -}} @@ -213,9 +304,9 @@ {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} {{- $conf := (get (fromJson (include "redpanda.Storage.GetTieredStorageConfig" (dict "a" (list $s) ))) "r") -}} -{{- $_503_b_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list $conf "cloud_storage_enabled" (coalesce nil)) ))) "r") -}} -{{- $b := (index $_503_b_ok 0) -}} -{{- $ok := (index $_503_b_ok 1) -}} +{{- $_646_b_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list $conf "cloud_storage_enabled" (coalesce nil)) ))) "r") -}} +{{- $b := (index $_646_b_ok 0) -}} +{{- $ok := (index $_646_b_ok 1) -}} {{- $_is_returning = true -}} {{- (dict "r" (and $ok (get (fromJson (include "_shims.typeassertion" (dict "a" (list "bool" $b) ))) "r"))) | toJson -}} {{- break -}} @@ -260,21 +351,21 @@ {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} {{- $values := $dot.Values.AsMap -}} -{{- $_532_dir_5_ok_6 := (get (fromJson (include "_shims.typetest" (dict "a" (list "string" (index $values.config.node "cloud_storage_cache_directory") "") ))) "r") -}} -{{- $dir_5 := (index $_532_dir_5_ok_6 0) -}} -{{- $ok_6 := (index $_532_dir_5_ok_6 1) -}} -{{- if $ok_6 -}} +{{- $_675_dir_7_ok_8 := (get (fromJson (include "_shims.typetest" (dict "a" (list "string" (index $values.config.node "cloud_storage_cache_directory") "") ))) "r") -}} +{{- $dir_7 := (index $_675_dir_7_ok_8 0) -}} +{{- $ok_8 := (index $_675_dir_7_ok_8 1) -}} +{{- if $ok_8 -}} {{- $_is_returning = true -}} -{{- (dict "r" $dir_5) | toJson -}} +{{- (dict "r" $dir_7) | toJson -}} {{- break -}} {{- end -}} {{- $tieredConfig := (get (fromJson (include "redpanda.Storage.GetTieredStorageConfig" (dict "a" (list $values.storage) ))) "r") -}} -{{- $_541_dir_7_ok_8 := (get (fromJson (include "_shims.typetest" (dict "a" (list "string" (index $tieredConfig "cloud_storage_cache_directory") "") ))) "r") -}} -{{- $dir_7 := (index $_541_dir_7_ok_8 0) -}} -{{- $ok_8 := (index $_541_dir_7_ok_8 1) -}} -{{- if $ok_8 -}} +{{- $_684_dir_9_ok_10 := (get (fromJson (include "_shims.typetest" (dict "a" (list "string" (index $tieredConfig "cloud_storage_cache_directory") "") ))) "r") -}} +{{- $dir_9 := (index $_684_dir_9_ok_10 0) -}} +{{- $ok_10 := (index $_684_dir_9_ok_10 1) -}} +{{- if $ok_10 -}} {{- $_is_returning = true -}} -{{- (dict "r" $dir_7) | toJson -}} +{{- (dict "r" $dir_9) | toJson -}} {{- break -}} {{- end -}} {{- $_is_returning = true -}} @@ -371,9 +462,9 @@ {{- $result := (dict ) -}} {{- $s := (toJson $t) -}} {{- $tune := (fromJson $s) -}} -{{- $_767_m_ok := (get (fromJson (include "_shims.typetest" (dict "a" (list (printf "map[%s]%s" "string" "interface {}") $tune (coalesce nil)) ))) "r") -}} -{{- $m := (index $_767_m_ok 0) -}} -{{- $ok := (index $_767_m_ok 1) -}} +{{- $_910_m_ok := (get (fromJson (include "_shims.typetest" (dict "a" (list (printf "map[%s]%s" "string" "interface {}") $tune (coalesce nil)) ))) "r") -}} +{{- $m := (index $_910_m_ok 0) -}} +{{- $ok := (index $_910_m_ok 1) -}} {{- if (not $ok) -}} {{- $_is_returning = true -}} {{- (dict "r" (dict )) | toJson -}} @@ -509,10 +600,10 @@ {{- $seen := (dict ) -}} {{- $deduped := (coalesce nil) -}} {{- range $_, $item := $items -}} -{{- $_884___ok_9 := (get (fromJson (include "_shims.dicttest" (dict "a" (list $seen $item.key false) ))) "r") -}} -{{- $_ := (index $_884___ok_9 0) -}} -{{- $ok_9 := (index $_884___ok_9 1) -}} -{{- if $ok_9 -}} +{{- $_1027___ok_11 := (get (fromJson (include "_shims.dicttest" (dict "a" (list $seen $item.key false) ))) "r") -}} +{{- $_ := (index $_1027___ok_11 0) -}} +{{- $ok_11 := (index $_1027___ok_11 1) -}} +{{- if $ok_11 -}} {{- continue -}} {{- end -}} {{- $deduped = (concat (default (list ) $deduped) (list $item)) -}} @@ -564,9 +655,9 @@ {{- $name := (index .a 1) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} -{{- $_975_cert_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list $m $name (dict "enabled" (coalesce nil) "caEnabled" false "applyInternalDNSNames" (coalesce nil) "duration" "" "issuerRef" (coalesce nil) "secretRef" (coalesce nil) "clientSecretRef" (coalesce nil) )) ))) "r") -}} -{{- $cert := (index $_975_cert_ok 0) -}} -{{- $ok := (index $_975_cert_ok 1) -}} +{{- $_1118_cert_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list $m $name (dict "enabled" (coalesce nil) "caEnabled" false "applyInternalDNSNames" (coalesce nil) "duration" "" "issuerRef" (coalesce nil) "secretRef" (coalesce nil) "clientSecretRef" (coalesce nil) )) ))) "r") -}} +{{- $cert := (index $_1118_cert_ok 0) -}} +{{- $ok := (index $_1118_cert_ok 1) -}} {{- if (not $ok) -}} {{- $_ := (fail (printf "Certificate %q referenced, but not found in the tls.certs map" $name)) -}} {{- end -}} @@ -914,9 +1005,9 @@ {{- if $saslEnabled -}} {{- $_ := (set $internal "authentication_method" "http_basic") -}} {{- end -}} -{{- $am_10 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} -{{- if (ne $am_10 "") -}} -{{- $_ := (set $internal "authentication_method" $am_10) -}} +{{- $am_12 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} +{{- if (ne $am_12 "") -}} +{{- $_ := (set $internal "authentication_method" $am_12) -}} {{- end -}} {{- $result := (list $internal) -}} {{- range $k, $l := $l.external -}} @@ -927,9 +1018,9 @@ {{- if $saslEnabled -}} {{- $_ := (set $listener "authentication_method" "http_basic") -}} {{- end -}} -{{- $am_11 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} -{{- if (ne $am_11 "") -}} -{{- $_ := (set $listener "authentication_method" $am_11) -}} +{{- $am_13 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} +{{- if (ne $am_13 "") -}} +{{- $_ := (set $listener "authentication_method" $am_13) -}} {{- end -}} {{- $result = (concat (default (list ) $result) (list $listener)) -}} {{- end -}} @@ -1012,9 +1103,9 @@ {{- if (get (fromJson (include "redpanda.Auth.IsSASLEnabled" (dict "a" (list $auth) ))) "r") -}} {{- $_ := (set $internal "authentication_method" "sasl") -}} {{- end -}} -{{- $am_12 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} -{{- if (ne $am_12 "") -}} -{{- $_ := (set $internal "authentication_method" $am_12) -}} +{{- $am_14 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} +{{- if (ne $am_14 "") -}} +{{- $_ := (set $internal "authentication_method" $am_14) -}} {{- end -}} {{- $kafka := (list $internal) -}} {{- range $k, $l := $l.external -}} @@ -1025,9 +1116,9 @@ {{- if (get (fromJson (include "redpanda.Auth.IsSASLEnabled" (dict "a" (list $auth) ))) "r") -}} {{- $_ := (set $listener "authentication_method" "sasl") -}} {{- end -}} -{{- $am_13 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} -{{- if (ne $am_13 "") -}} -{{- $_ := (set $listener "authentication_method" $am_13) -}} +{{- $am_15 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} +{{- if (ne $am_15 "") -}} +{{- $_ := (set $listener "authentication_method" $am_15) -}} {{- end -}} {{- $kafka = (concat (default (list ) $kafka) (list $listener)) -}} {{- end -}} @@ -1159,9 +1250,9 @@ {{- if $saslEnabled -}} {{- $_ := (set $internal "authentication_method" "http_basic") -}} {{- end -}} -{{- $am_14 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $sr.authenticationMethod "") ))) "r") -}} -{{- if (ne $am_14 "") -}} -{{- $_ := (set $internal "authentication_method" $am_14) -}} +{{- $am_16 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $sr.authenticationMethod "") ))) "r") -}} +{{- if (ne $am_16 "") -}} +{{- $_ := (set $internal "authentication_method" $am_16) -}} {{- end -}} {{- $result := (list $internal) -}} {{- range $k, $l := $sr.external -}} @@ -1172,9 +1263,9 @@ {{- if $saslEnabled -}} {{- $_ := (set $listener "authentication_method" "http_basic") -}} {{- end -}} -{{- $am_15 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} -{{- if (ne $am_15 "") -}} -{{- $_ := (set $listener "authentication_method" $am_15) -}} +{{- $am_17 := (get (fromJson (include "_shims.ptr_Deref" (dict "a" (list $l.authenticationMethod "") ))) "r") -}} +{{- if (ne $am_17 "") -}} +{{- $_ := (set $listener "authentication_method" $am_17) -}} {{- end -}} {{- $result = (concat (default (list ) $result) (list $listener)) -}} {{- end -}} @@ -1309,10 +1400,10 @@ {{- $result := (dict ) -}} {{- range $k, $v := $c -}} {{- if (not (empty $v)) -}} -{{- $_1809___ok_16 := (get (fromJson (include "_shims.asnumeric" (dict "a" (list $v) ))) "r") -}} -{{- $_ := ((index $_1809___ok_16 0) | float64) -}} -{{- $ok_16 := (index $_1809___ok_16 1) -}} -{{- if $ok_16 -}} +{{- $_1952___ok_18 := (get (fromJson (include "_shims.asnumeric" (dict "a" (list $v) ))) "r") -}} +{{- $_ := ((index $_1952___ok_18 0) | float64) -}} +{{- $ok_18 := (index $_1952___ok_18 1) -}} +{{- if $ok_18 -}} {{- $_ := (set $result $k $v) -}} {{- else -}}{{- if (kindIs "bool" $v) -}} {{- $_ := (set $result $k $v) -}} @@ -1337,11 +1428,11 @@ {{- $_is_returning := false -}} {{- $result := (dict ) -}} {{- range $k, $v := $c -}} -{{- $_1829_b_17_ok_18 := (get (fromJson (include "_shims.typetest" (dict "a" (list "bool" $v false) ))) "r") -}} -{{- $b_17 := (index $_1829_b_17_ok_18 0) -}} -{{- $ok_18 := (index $_1829_b_17_ok_18 1) -}} -{{- if $ok_18 -}} -{{- $_ := (set $result $k $b_17) -}} +{{- $_1972_b_19_ok_20 := (get (fromJson (include "_shims.typetest" (dict "a" (list "bool" $v false) ))) "r") -}} +{{- $b_19 := (index $_1972_b_19_ok_20 0) -}} +{{- $ok_20 := (index $_1972_b_19_ok_20 1) -}} +{{- if $ok_20 -}} +{{- $_ := (set $result $k $b_19) -}} {{- continue -}} {{- end -}} {{- if (not (empty $v)) -}} @@ -1382,15 +1473,15 @@ {{- $config := (index .a 1) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} -{{- $_1874___hasAccessKey := (get (fromJson (include "_shims.dicttest" (dict "a" (list $config "cloud_storage_access_key" (coalesce nil)) ))) "r") -}} -{{- $_ := (index $_1874___hasAccessKey 0) -}} -{{- $hasAccessKey := (index $_1874___hasAccessKey 1) -}} -{{- $_1875___hasSecretKey := (get (fromJson (include "_shims.dicttest" (dict "a" (list $config "cloud_storage_secret_key" (coalesce nil)) ))) "r") -}} -{{- $_ := (index $_1875___hasSecretKey 0) -}} -{{- $hasSecretKey := (index $_1875___hasSecretKey 1) -}} -{{- $_1876___hasSharedKey := (get (fromJson (include "_shims.dicttest" (dict "a" (list $config "cloud_storage_azure_shared_key" (coalesce nil)) ))) "r") -}} -{{- $_ := (index $_1876___hasSharedKey 0) -}} -{{- $hasSharedKey := (index $_1876___hasSharedKey 1) -}} +{{- $_2017___hasAccessKey := (get (fromJson (include "_shims.dicttest" (dict "a" (list $config "cloud_storage_access_key" (coalesce nil)) ))) "r") -}} +{{- $_ := (index $_2017___hasAccessKey 0) -}} +{{- $hasAccessKey := (index $_2017___hasAccessKey 1) -}} +{{- $_2018___hasSecretKey := (get (fromJson (include "_shims.dicttest" (dict "a" (list $config "cloud_storage_secret_key" (coalesce nil)) ))) "r") -}} +{{- $_ := (index $_2018___hasSecretKey 0) -}} +{{- $hasSecretKey := (index $_2018___hasSecretKey 1) -}} +{{- $_2019___hasSharedKey := (get (fromJson (include "_shims.dicttest" (dict "a" (list $config "cloud_storage_azure_shared_key" (coalesce nil)) ))) "r") -}} +{{- $_ := (index $_2019___hasSharedKey 0) -}} +{{- $hasSharedKey := (index $_2019___hasSharedKey 1) -}} {{- $envvars := (coalesce nil) -}} {{- if (and (not $hasAccessKey) (get (fromJson (include "redpanda.SecretRef.IsValid" (dict "a" (list $tsc.accessKey) ))) "r")) -}} {{- $envvars = (concat (default (list ) $envvars) (list (mustMergeOverwrite (dict "name" "" ) (dict "name" "REDPANDA_CLOUD_STORAGE_ACCESS_KEY" "valueFrom" (get (fromJson (include "redpanda.SecretRef.AsSource" (dict "a" (list $tsc.accessKey) ))) "r") )))) -}} @@ -1413,12 +1504,12 @@ {{- $c := (index .a 0) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} -{{- $_1912___containerExists := (get (fromJson (include "_shims.dicttest" (dict "a" (list $c "cloud_storage_azure_container" (coalesce nil)) ))) "r") -}} -{{- $_ := (index $_1912___containerExists 0) -}} -{{- $containerExists := (index $_1912___containerExists 1) -}} -{{- $_1913___accountExists := (get (fromJson (include "_shims.dicttest" (dict "a" (list $c "cloud_storage_azure_storage_account" (coalesce nil)) ))) "r") -}} -{{- $_ := (index $_1913___accountExists 0) -}} -{{- $accountExists := (index $_1913___accountExists 1) -}} +{{- $_2055___containerExists := (get (fromJson (include "_shims.dicttest" (dict "a" (list $c "cloud_storage_azure_container" (coalesce nil)) ))) "r") -}} +{{- $_ := (index $_2055___containerExists 0) -}} +{{- $containerExists := (index $_2055___containerExists 1) -}} +{{- $_2056___accountExists := (get (fromJson (include "_shims.dicttest" (dict "a" (list $c "cloud_storage_azure_storage_account" (coalesce nil)) ))) "r") -}} +{{- $_ := (index $_2056___accountExists 0) -}} +{{- $accountExists := (index $_2056___accountExists 1) -}} {{- $_is_returning = true -}} {{- (dict "r" (and $containerExists $accountExists)) | toJson -}} {{- break -}} @@ -1429,9 +1520,9 @@ {{- $c := (index .a 0) -}} {{- range $_ := (list 1) -}} {{- $_is_returning := false -}} -{{- $_1918_value_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list $c `cloud_storage_cache_size` (coalesce nil)) ))) "r") -}} -{{- $value := (index $_1918_value_ok 0) -}} -{{- $ok := (index $_1918_value_ok 1) -}} +{{- $_2061_value_ok := (get (fromJson (include "_shims.dicttest" (dict "a" (list $c `cloud_storage_cache_size` (coalesce nil)) ))) "r") -}} +{{- $value := (index $_2061_value_ok 0) -}} +{{- $ok := (index $_2061_value_ok 1) -}} {{- if (not $ok) -}} {{- $_is_returning = true -}} {{- (dict "r" (coalesce nil)) | toJson -}} @@ -1456,9 +1547,9 @@ {{- if $_is_returning -}} {{- break -}} {{- end -}} -{{- $size_19 := (get (fromJson (include "redpanda.TieredStorageConfig.CloudStorageCacheSize" (dict "a" (list (deepCopy $c)) ))) "r") -}} -{{- if (ne (toJson $size_19) "null") -}} -{{- $_ := (set $config "cloud_storage_cache_size" ((get (fromJson (include "_shims.resource_Value" (dict "a" (list $size_19) ))) "r") | int64)) -}} +{{- $size_21 := (get (fromJson (include "redpanda.TieredStorageConfig.CloudStorageCacheSize" (dict "a" (list (deepCopy $c)) ))) "r") -}} +{{- if (ne (toJson $size_21) "null") -}} +{{- $_ := (set $config "cloud_storage_cache_size" ((get (fromJson (include "_shims.resource_Value" (dict "a" (list $size_21) ))) "r") | int64)) -}} {{- end -}} {{- $_is_returning = true -}} {{- (dict "r" $config) | toJson -}} diff --git a/charts/redpanda/testdata/template-cases.txtar b/charts/redpanda/testdata/template-cases.txtar index 347052a412..dd82ecf2de 100644 --- a/charts/redpanda/testdata/template-cases.txtar +++ b/charts/redpanda/testdata/template-cases.txtar @@ -651,3 +651,27 @@ console: hosts: - host: "{{ (get (fromJson (include \"console.Fullname\" (dict \"a\" (list $) ))) \"r\") | trunc 50 }}-first-rule-host" - host: "{{ (get (fromJson (include \"console.Fullname\" (dict \"a\" (list $) ))) \"r\") | trunc 50 }}-second-rule-host" + +-- direct-resources-examples -- +# ASSERT-NO-ERROR +# ASSERT-FIELD-EQUALS ["apps/v1/StatefulSet", "default/redpanda", "{.spec.template.spec.containers[0].resources.limits}", {"memory": "10Gi"}] +# ASSERT-FIELD-EQUALS ["apps/v1/StatefulSet", "default/redpanda", "{.spec.template.spec.containers[0].resources.requests}", {"cpu": "5500m"}] +# ASSERT-FIELD-CONTAINS ["v1/ConfigMap", "default/redpanda", "{.data.redpanda\\.yaml}", "--smp=5"] +# ASSERT-FIELD-CONTAINS ["v1/ConfigMap", "default/redpanda", "{.data.redpanda\\.yaml}", "--memory=9216M"] +# ASSERT-FIELD-CONTAINS ["v1/ConfigMap", "default/redpanda", "{.data.redpanda\\.yaml}", "--reserve-memory=0M"] +resources: + limits: + memory: 10Gi + requests: + cpu: 5500m + +-- direct-resources-none-provided -- +# ASSERT-NO-ERROR +# ASSERT-FIELD-EQUALS ["apps/v1/StatefulSet", "default/redpanda", "{.spec.template.spec.containers[0].resources.limits}", {}] +# ASSERT-FIELD-EQUALS ["apps/v1/StatefulSet", "default/redpanda", "{.spec.template.spec.containers[0].resources.requests}", {}] +# ASSERT-FIELD-NOT-CONTAINS ["v1/ConfigMap", "default/redpanda", "{.data.redpanda\\.yaml}", "--smp"] +# ASSERT-FIELD-NOT-CONTAINS ["v1/ConfigMap", "default/redpanda", "{.data.redpanda\\.yaml}", "--memory"] +# ASSERT-FIELD-CONTAINS ["v1/ConfigMap", "default/redpanda", "{.data.redpanda\\.yaml}", "--reserve-memory=0M"] +resources: + limits: {} + requests: {} diff --git a/charts/redpanda/values.go b/charts/redpanda/values.go index 2c7d632e50..689dc842a2 100644 --- a/charts/redpanda/values.go +++ b/charts/redpanda/values.go @@ -295,7 +295,25 @@ type Monitoring struct { EnableHttp2 *bool `json:"enableHttp2"` } +// RedpandaResources encapsulates the calculation of the redpanda container's +// [corev1.ResourceRequirements] and parameters such as `--memory`, +// `--reserve-memory`, and `--smp`. +// This calculation occurs in two "modes" +// * "Implied" - When Limits and Requests are set. +// * "Historic" - When Limits and Requests are not set. +// +// Implied mode (recommended) will respect the Limits and Requests as given and +// calculate redpanda's CLI flags based off what's provided. Should additional +// tuning be required, flags may be set directly through their appropriate +// values.yaml option. +// +// Historic mode is the default which introduced a bespoke modeling of both +// redpanda's CLI flags and container resources. See it's substructs for +// further explanation. type RedpandaResources struct { + Limits *corev1.ResourceList `json:"limits,omitempty"` + Requests *corev1.ResourceList `json:"requests,omitempty"` + CPU struct { Cores resource.Quantity `json:"cores" jsonschema:"required"` Overprovisioned *bool `json:"overprovisioned"` @@ -363,6 +381,14 @@ type RedpandaResources struct { } func (rr *RedpandaResources) GetResourceRequirements() corev1.ResourceRequirements { + // If Limits and Requests are specified, use them as is. + if rr.Limits != nil && rr.Requests != nil { + return corev1.ResourceRequirements{ + Limits: *rr.Limits, + Requests: *rr.Requests, + } + } + // Otherwise fallback to the historical behavior. reqs := corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -381,22 +407,49 @@ func (rr *RedpandaResources) GetResourceRequirements() corev1.ResourceRequiremen return reqs } -func (rr *RedpandaResources) GetRedpandaStartFlags() map[string]string { - flags := map[string]string{} +func (rr *RedpandaResources) GetRedpandaFlags() map[string]string { + flags := map[string]string{ + "reserve-memory": fmt.Sprintf("%dM", rr.reserveMemory()), + } - if coresInMillies := rr.CPU.Cores.MilliValue(); coresInMillies < 1000 { - flags["smp"] = fmt.Sprintf("%d", 1) - } else { - flags["smp"] = fmt.Sprintf("%d", rr.CPU.Cores.Value()) + if smp := rr.smp(); smp != nil { + flags["smp"] = fmt.Sprintf("%d", int64(*smp)) } - flags["memory"] = fmt.Sprintf("%dM", rr.memory()) - flags["reserve-memory"] = fmt.Sprintf("%dM", rr.reserveMemory()) + if memory := rr.memory(); memory != nil { + flags["memory"] = fmt.Sprintf("%dM", int64(*memory)) + } + + // Only set lock-memory if Limits and Requests are NOT specified. It should + // otherwise be set through additionalRedpandaCmdFlags. + if rr.Limits == nil && rr.Requests == nil { + flags["lock-memory"] = fmt.Sprintf("%v", ptr.Deref(rr.Memory.EnableMemoryLocking, false)) + } + + if rr.GetOverProvisionValue() { + flags["overprovisioned"] = "" + } return flags } func (rr *RedpandaResources) GetOverProvisionValue() bool { + if rr.Limits != nil && rr.Requests != nil { + // Get CPU prioritizing requests, falling back to limits if not + // specified as kube-scheduler does. + cpuReq, ok := (*rr.Requests)[corev1.ResourceCPU] + if !ok { + cpuReq, ok = (*rr.Limits)[corev1.ResourceCPU] + } + + // If redpanda has been allocated less than 1 full CPU, set + // overprovisioned to true. + if ok && cpuReq.MilliValue() < 1000 { + return true + } + return false + } + if rr.CPU.Cores.MilliValue() < 1000 { return true } @@ -404,12 +457,97 @@ func (rr *RedpandaResources) GetOverProvisionValue() bool { return ptr.Deref(rr.CPU.Overprovisioned, false) } +func (rr *RedpandaResources) smp() *int64 { + if rr.Limits != nil && rr.Requests != nil { + // Get CPU prioritizing requests, falling back to limits if not + // specified as kube-scheduler does. + cpuReq, ok := (*rr.Requests)[corev1.ResourceCPU] + if !ok { + cpuReq, ok = (*rr.Limits)[corev1.ResourceCPU] + } + + // If neither requests nor limits are set, don't set --smp. + if !ok { + return nil + } + + // If CPU requests are defined, set --smp to min(1, floor(requests.cpu)). + // + // Due to redpanda/seastar's per core model, we can't do much with + // fractional CPU values we need to round either up or down. Rounding + // up would result in utilizing too much CPU from the CRI perspective + // and cause throttling, so we round down and potentially waste some + // quota. + smp := cpuReq.MilliValue() / 1000 + if smp < 1 { + smp = 1 + } + return ptr.To(smp) + } + + if coresInMillies := rr.CPU.Cores.MilliValue(); coresInMillies < 1000 { + return ptr.To(int64(1)) + } + return ptr.To(int64(rr.CPU.Cores.Value())) +} + // memory returns the amount of memory for Redpanda process. It should be // passed to the `--memory` argument of the Redpanda process, see // RedpandaAdditionalStartFlags and rpk redpanda start documentation. // // https://docs.redpanda.com/current/reference/rpk/rpk-redpanda/rpk-redpanda-start/ -func (rr *RedpandaResources) memory() int64 { +func (rr *RedpandaResources) memory() *MebiBytes { + if rr.Limits != nil && rr.Requests != nil { + // `--memory` will be set to something < the container's + // resources.memory.limits value. + // We want to allocate seastar < memory than our limit for several reasons: + // 1. Seastar may slightly exceed this limit due to page tables and + // non-heap memory that's still accounted by cgroups. + // 2. resources.limits.memory applies to the entire container. We want + // to keep headroom to allow exec'ing into the container and for any + // exec probes. + // 3. emptyDir's storage is counted against the container's memory + // limits. We use these to store rendered versions of config files + // and therefore need to account for them. + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#memory-backed-emptydir + // The memory reservation is done by subtracting from `--memory` and + // always setting `--reserve-memory` to 0 rather than setting + // `--reserve-memory`. This is an easier mental model to follow as the + // `--reserve-memory` flag is exceptionally nuanced in practice. + + // If either memory limit or requests are set, we take the minimum of + // the two (relying on the invariant enforced by Kubernetes that + // requests must be <= limits). + memReq, ok := (*rr.Requests)[corev1.ResourceMemory] + if !ok { + memReq, ok = (*rr.Limits)[corev1.ResourceMemory] + } + + // If neither requests nor limits are set, don't set --memory. + if !ok { + return nil + } + + // Here we perform out memory reservation. Historically, the intention + // was to reserve 0.2%+200Mi of memory but an unnoticed bug resulted in + // reserving 20%. So we're largely blind to the lower limit of what's + // tolerated. + // This calculation, therefore, is a complete split ball. We expect + // this to change over time; ideally trending towards providing + // redpanda more memory. + // We intentionally err on the conservative side as we'd prefer to + // "waste" a few megs of memory rather than risking OOM kills. + // For simplicity, we're using a % as a static reservation would need + // to handle weird edge cases. + // + // redpanda get's 90% of the container limit. (It's better than the + // historic 80%). + memory := int64(float64(memReq.Value()) * 0.90) + + // Cast to Membibytes. + return ptr.To(memory / (1024 * 1024)) + } + memory := int64(0) containerMemory := rr.containerMemory() @@ -434,11 +572,13 @@ func (rr *RedpandaResources) memory() int64 { panic(fmt.Sprintf("%d is below the minimum value for Redpanda", memory)) } - if memory+rr.reserveMemory() > containerMemory { + // NB: int64's are working around a bug in gotohelm's BinaryExpr detection + // with Alias types. + if memory+int64(rr.reserveMemory()) > containerMemory { panic(fmt.Sprintf("Not enough container memory for Redpanda memory values where Redpanda: %d, reserve: %d, container: %d", memory, rr.reserveMemory(), containerMemory)) } - return memory + return ptr.To(memory) } // reserveMemory returns the amount of memory that the Redpanda process will @@ -448,7 +588,12 @@ func (rr *RedpandaResources) memory() int64 { // start documentation. // // https://docs.redpanda.com/current/reference/rpk/rpk-redpanda/rpk-redpanda-start/ -func (rr *RedpandaResources) reserveMemory() int64 { +func (rr *RedpandaResources) reserveMemory() MebiBytes { + if rr.Limits != nil && rr.Requests != nil { + // See [RedpandaResources.memory] for details here. + return 0 + } + // This optional `redpanda` object allows you to specify the memory size for both the Redpanda // process and the underlying reserved memory used by Seastar. // diff --git a/charts/redpanda/values.schema.json b/charts/redpanda/values.schema.json index 8c90d45c3c..2f6f3b9b73 100644 --- a/charts/redpanda/values.schema.json +++ b/charts/redpanda/values.schema.json @@ -14013,6 +14013,20 @@ ], "type": "object" }, + "limits": { + "additionalProperties": { + "oneOf": [ + { + "type": "integer" + }, + { + "pattern": "^[0-9]+(\\.[0-9]){0,1}(m|k|M|G|T|P|Ki|Mi|Gi|Ti|Pi)?$", + "type": "string" + } + ] + }, + "type": "object" + }, "memory": { "properties": { "container": { @@ -14080,6 +14094,20 @@ "container" ], "type": "object" + }, + "requests": { + "additionalProperties": { + "oneOf": [ + { + "type": "integer" + }, + { + "pattern": "^[0-9]+(\\.[0-9]){0,1}(m|k|M|G|T|P|Ki|Mi|Gi|Ti|Pi)?$", + "type": "string" + } + ] + }, + "type": "object" } }, "required": [ diff --git a/charts/redpanda/values.yaml b/charts/redpanda/values.yaml index 00192519b8..e0a94c57b3 100644 --- a/charts/redpanda/values.yaml +++ b/charts/redpanda/values.yaml @@ -346,9 +346,35 @@ monitoring: # keyFile: /etc/prom-certs/key.pem # -- Pod resource management. -# This section simplifies resource allocation -# by providing a single location where resources are defined. -# Helm sets these resource values within the `statefulset.yaml` and `configmap.yaml` templates. +# This section simplifies resource allocation for the redpanda container by +# providing a single location where resources are defined. +# +# Resources may be specified by either setting `resources.cpu` and +# `resources.memory` (the default) or by setting `resources.requests` and +# `resources.limits`. +# +# For details on `resources.cpu` and `resources.memory`, see their respective +# documentation below. +# +# When `resources.limits` and `resources.requests` are set, the redpanda +# container's resources will be set to exactly the provided values. This allows +# users to granularly control limits and requests to best suite their use case. +# For example: `resources.requests.cpu` may be set without setting +# `resources.limits.cpu` to avoid the potential of CPU throttling. +# +# Redpanda's resource related CLI flags will then be calculated as follows: +# * `--smp floor(resources.{requests,limits}.cpu)` +# * `--memory resources.{requests,limits}.memory * 90%` +# * `--reserve-memory 0` +# * `--overprovisioned resources.{requests,limits}.cpu < 1000m` +# +# If neither a request nor a limit is for cpu or memory provided, the +# corresponding flag will be omitted. i.e. Setting `resources.limits` and +# `resources.requests` to `{}` will result in redpanda being run without +# `--smp` or `--memory`. (This is not recommended). +# +# If the computed CLI flags are undesirable, they may be overridden by +# specifying the desired value through `statefulset.additionalRedpandaCmdFlags`. # # The default values are for a development environment. # Production-level values and other considerations are documented, @@ -356,6 +382,8 @@ monitoring: # For details, # see the [Pod resources documentation](https://docs.redpanda.com/docs/manage/kubernetes/manage-resources/). resources: + # limits: null + # requests: null # # -- CPU resources. # For details, diff --git a/charts/redpanda/values_partial.gen.go b/charts/redpanda/values_partial.gen.go index 1927d33d57..61326ea727 100644 --- a/charts/redpanda/values_partial.gen.go +++ b/charts/redpanda/values_partial.gen.go @@ -123,7 +123,9 @@ type PartialMonitoring struct { } type PartialRedpandaResources struct { - CPU *struct { + Limits *corev1.ResourceList "json:\"limits,omitempty\"" + Requests *corev1.ResourceList "json:\"requests,omitempty\"" + CPU *struct { Cores *resource.Quantity "json:\"cores,omitempty\" jsonschema:\"required\"" Overprovisioned *bool "json:\"overprovisioned,omitempty\"" } "json:\"cpu,omitempty\" jsonschema:\"required\"" diff --git a/charts/redpanda/values_test.go b/charts/redpanda/values_test.go index 29939fa852..a7d6856d31 100644 --- a/charts/redpanda/values_test.go +++ b/charts/redpanda/values_test.go @@ -4,8 +4,10 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/ptr" ) @@ -609,3 +611,87 @@ func TestTieredStorageConfigCreds(t *testing.T) { }) } } + +func TestRedpandaResources_RedpandaFlags(t *testing.T) { + cases := []struct { + Resources RedpandaResources + Expected map[string]string + }{ + { + Resources: RedpandaResources{ + Limits: &corev1.ResourceList{}, + Requests: &corev1.ResourceList{}, + }, + Expected: map[string]string{ + "reserve-memory": "0M", // Always set when Limits && Requests != nil. + // No other flags set as there's nothing to base them off of (Not recommended). + }, + }, + { + // overprovisioned is only set if CPU < 1000m. + Resources: RedpandaResources{ + Limits: &corev1.ResourceList{}, + Requests: &corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + }, + }, + Expected: map[string]string{ + "reserve-memory": "0M", // Always set when Limits && Requests != nil. + "smp": "1", + "overprovisioned": "", + }, + }, + { + Resources: RedpandaResources{ + Limits: &corev1.ResourceList{}, + Requests: &corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2500m"), + corev1.ResourceMemory: resource.MustParse("10Gi"), + }, + }, + Expected: map[string]string{ + "reserve-memory": "0M", + "smp": "2", // floor(CPU) + "memory": "9216M", // memory * 90% + }, + }, + { + // Limits are taken if requests aren't specified. + Resources: RedpandaResources{ + Limits: &corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + corev1.ResourceMemory: resource.MustParse("20Gi"), + }, + Requests: &corev1.ResourceList{}, + }, + Expected: map[string]string{ + "reserve-memory": "0M", + "smp": "3", // floor(CPU) + "memory": "18432M", // memory * 90% + }, + }, + { + // Showcase that Requests are taken for CLI params in favor of limits. + Resources: RedpandaResources{ + Limits: &corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10"), + corev1.ResourceMemory: resource.MustParse("200Gi"), + }, + Requests: &corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourceMemory: resource.MustParse("100Gi"), + }, + }, + Expected: map[string]string{ + "reserve-memory": "0M", + "smp": "5", // floor(CPU) + "memory": "92160M", // memory * 90% + }, + }, + } + + for _, tc := range cases { + flags := tc.Resources.GetRedpandaFlags() + assert.Equal(t, tc.Expected, flags) + } +} diff --git a/flake.nix b/flake.nix index 41e86426a5..fae1a04e28 100644 --- a/flake.nix +++ b/flake.nix @@ -55,7 +55,7 @@ pkgs.kubernetes-helm pkgs.kustomize pkgs.setup-envtest - pkgs.yq # jq but for YAML + pkgs.yq-go # jq but for YAML ]; }; };