diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index 3c09c29..7c01305 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: karpenter description: A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. type: application -version: 0.37.2 -appVersion: 0.37.0 +version: 1.1.0 +appVersion: 1.1.0 keywords: - cluster - node diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md index 152c462..58669e7 100644 --- a/charts/karpenter/README.md +++ b/charts/karpenter/README.md @@ -2,7 +2,7 @@ A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.37.0](https://img.shields.io/badge/AppVersion-0.37.0-informational?style=flat-square) +![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.0](https://img.shields.io/badge/AppVersion-1.1.0-informational?style=flat-square) ## Documentation @@ -15,7 +15,7 @@ You can follow the detailed installation instruction in the [documentation](http ```bash helm upgrade --install --namespace karpenter --create-namespace \ karpenter oci://public.ecr.aws/karpenter/karpenter \ - --version 0.37.0 \ + --version 1.1.0 \ --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}" \ --set settings.clusterName=${CLUSTER_NAME} \ --set settings.interruptionQueue=${CLUSTER_NAME} \ @@ -27,13 +27,13 @@ helm upgrade --install --namespace karpenter --create-namespace \ As the OCI Helm chart is signed by [Cosign](https://github.com/sigstore/cosign) as part of the release process you can verify the chart before installing it by running the following command. ```shell -cosign verify public.ecr.aws/karpenter/karpenter:0.37.0 \ +cosign verify public.ecr.aws/karpenter/karpenter:1.1.0 \ --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ --certificate-identity-regexp='https://github\.com/aws/karpenter-provider-aws/\.github/workflows/release\.yaml@.+' \ --certificate-github-workflow-repository=aws/karpenter-provider-aws \ --certificate-github-workflow-name=Release \ - --certificate-github-workflow-ref=refs/tags/v0.37.0 \ - --annotations version=0.37.0 + --certificate-github-workflow-ref=refs/tags/v1.1.0 \ + --annotations version=1.1.0 ``` ## Values @@ -44,14 +44,15 @@ cosign verify public.ecr.aws/karpenter/karpenter:0.37.0 \ | additionalClusterRoleRules | list | `[]` | Specifies additional rules for the core ClusterRole. | | additionalLabels | object | `{}` | Additional labels to add into metadata. | | affinity | object | `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"karpenter.sh/nodepool","operator":"DoesNotExist"}]}]}},"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity rules for scheduling the pod. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels. | +| controller.containerName | string | `"controller"` | Distinguishing container name (containerName: karpenter-controller). | | controller.env | list | `[]` | Additional environment variables for the controller pod. | | controller.envFrom | list | `[]` | | | controller.extraVolumeMounts | list | `[]` | Additional volumeMounts for the controller pod. | | controller.healthProbe.port | int | `8081` | The container port to use for http health probe. | -| controller.image.digest | string | `"sha256:157f478f5db1fe999f5e2d27badcc742bf51cc470508b3cebe78224d0947674f"` | SHA256 digest of the controller image. | +| controller.image.digest | string | `"sha256:51bca600197c7c6e6e0838549664b2c12c3f8dd4b23744ab28202ae97ca5aed1"` | SHA256 digest of the controller image. | | controller.image.repository | string | `"public.ecr.aws/karpenter/controller"` | Repository path to the controller image. | -| controller.image.tag | string | `"0.37.0"` | Tag of the controller image. | -| controller.metrics.port | int | `8000` | The container port to use for metrics. | +| controller.image.tag | string | `"1.1.0"` | Tag of the controller image. | +| controller.metrics.port | int | `8080` | The container port to use for metrics. | | controller.resources | object | `{}` | Resources for the controller pod. | | controller.sidecarContainer | list | `[]` | Additional sidecarContainer config | | controller.sidecarVolumeMounts | list | `[]` | Additional volumeMounts for the sidecar - this will be added to the volume mounts on top of extraVolumeMounts | @@ -62,16 +63,9 @@ cosign verify public.ecr.aws/karpenter/karpenter:0.37.0 \ | hostNetwork | bool | `false` | Bind the pod to the host network. This is required when using a custom CNI. | | imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for Docker images. | | imagePullSecrets | list | `[]` | Image pull secrets for Docker images. | -| logConfig | object | `{"enabled":false,"errorOutputPaths":["stderr"],"logEncoding":"json","logLevel":{"controller":"info","global":"info","webhook":"error"},"outputPaths":["stdout"]}` | Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) | -| logConfig.enabled | bool | `false` | Whether to enable provisioning and mounting the log ConfigMap | -| logConfig.errorOutputPaths | list | `["stderr"]` | Log errorOutputPaths - defaults to stderr only | -| logConfig.logEncoding | string | `"json"` | Log encoding - defaults to json - must be one of 'json', 'console' | -| logConfig.logLevel | object | `{"controller":"info","global":"info","webhook":"error"}` | Component-based log configuration | -| logConfig.logLevel.controller | string | `"info"` | Controller log level, defaults to 'info' | -| logConfig.logLevel.global | string | `"info"` | Global log level, defaults to 'info' | -| logConfig.logLevel.webhook | string | `"error"` | Error log level, defaults to 'error' | -| logConfig.outputPaths | list | `["stdout"]` | Log outputPaths - defaults to stdout only | +| logErrorOutputPaths | list | `["stderr"]` | Log errorOutputPaths - defaults to stderr only | | logLevel | string | `"info"` | Global log level, defaults to 'info' | +| logOutputPaths | list | `["stdout"]` | Log outputPaths - defaults to stdout only | | nameOverride | string | `""` | Overrides the chart's name. | | nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selectors to schedule the pod to nodes with labels. | | podAnnotations | object | `{}` | Additional annotations for the pod. | @@ -82,34 +76,31 @@ cosign verify public.ecr.aws/karpenter/karpenter:0.37.0 \ | priorityClassName | string | `"system-cluster-critical"` | PriorityClass name for the pod. | | replicas | int | `2` | Number of replicas. | | revisionHistoryLimit | int | `10` | The number of old ReplicaSets to retain to allow rollback. | +| service.annotations | object | `{}` | Additional annotations for the Service. | | serviceAccount.annotations | object | `{}` | Additional annotations for the ServiceAccount. | | serviceAccount.create | bool | `true` | Specifies if a ServiceAccount should be created. | | serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | | serviceMonitor.additionalLabels | object | `{}` | Additional labels for the ServiceMonitor. | | serviceMonitor.enabled | bool | `false` | Specifies whether a ServiceMonitor should be created. | -| serviceMonitor.endpointConfig | object | `{}` | Configuration on `http-metrics` endpoint for the ServiceMonitor. Not to be used to add additional endpoints. See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint | -| settings | object | `{"assumeRoleARN":"","assumeRoleDuration":"15m","batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","featureGates":{"drift":true,"spotToSpotConsolidation":false},"interruptionQueue":"","isolatedVPC":false,"reservedENIs":"0","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter | -| settings.assumeRoleARN | string | `""` | Role to assume for calling AWS services. | -| settings.assumeRoleDuration | string | `"15m"` | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless assumeRoleARN set. | +| serviceMonitor.endpointConfig | object | `{}` | Configuration on `http-metrics` endpoint for the ServiceMonitor. Not to be used to add additional endpoints. See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint | +| settings | object | `{"batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","eksControlPlane":false,"featureGates":{"nodeRepair":false,"spotToSpotConsolidation":false},"interruptionQueue":"","isolatedVPC":false,"reservedENIs":"0","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter | | settings.batchIdleDuration | string | `"1s"` | The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. | | settings.batchMaxDuration | string | `"10s"` | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. | | settings.clusterCABundle | string | `""` | Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. | | settings.clusterEndpoint | string | `""` | Cluster endpoint. If not set, will be discovered during startup (EKS only) | | settings.clusterName | string | `""` | Cluster name. | -| settings.featureGates | object | `{"drift":true,"spotToSpotConsolidation":false}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features | -| settings.featureGates.drift | bool | `true` | drift is in BETA and is enabled by default. Setting drift to false disables the drift disruption method to watch for drift between currently deployed nodes and the desired state of nodes set in nodepools and nodeclasses | +| settings.eksControlPlane | bool | `false` | Marking this true means that your cluster is running with an EKS control plane and Karpenter should attempt to discover cluster details from the DescribeCluster API | +| settings.featureGates | object | `{"nodeRepair":false,"spotToSpotConsolidation":false}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features | +| settings.featureGates.nodeRepair | bool | `false` | nodeRepair is ALPHA and is disabled by default. Setting this to true will enable node repair. | | settings.featureGates.spotToSpotConsolidation | bool | `false` | spotToSpotConsolidation is ALPHA and is disabled by default. Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. | | settings.interruptionQueue | string | `""` | Interruption queue is the name of the SQS queue used for processing interruption events from EC2 Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs. | | settings.isolatedVPC | bool | `false` | If true then assume we can't reach AWS services which don't have a VPC endpoint This also has the effect of disabling look-ups to the AWS pricing endpoint | | settings.reservedENIs | string | `"0"` | Reserved ENIs are not included in the calculations for max-pods or kube-reserved This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html | -| settings.vmMemoryOverheadPercent | float | `0.075` | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types | +| settings.vmMemoryOverheadPercent | float | `0.075` | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. The value of `0.075` equals to 7.5%. | | strategy | object | `{"rollingUpdate":{"maxUnavailable":1}}` | Strategy for updating the pod. | | terminationGracePeriodSeconds | string | `nil` | Override the default termination grace period for the pod. | | tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"}]` | Tolerations to allow the pod to be scheduled to nodes with taints. | -| topologySpreadConstraints | list | `[{"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone","whenUnsatisfiable":"ScheduleAnyway"}]` | Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. | -| webhook.enabled | bool | `false` | Whether to enable the webhooks and webhook permissions. | -| webhook.metrics.port | int | `8001` | The container port to use for webhook metrics. | -| webhook.port | int | `8443` | The container port to use for the webhook. | +| topologySpreadConstraints | list | `[{"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone","whenUnsatisfiable":"DoNotSchedule"}]` | Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. | ---------------------------------------------- diff --git a/charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index aec3d01..97917a0 100644 --- a/charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -3,622 +3,740 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.5 name: ec2nodeclasses.karpenter.k8s.aws spec: group: karpenter.k8s.aws names: categories: - - karpenter + - karpenter kind: EC2NodeClass listKind: EC2NodeClassList plural: ec2nodeclasses shortNames: - - ec2nc - - ec2ncs + - ec2nc + - ec2ncs singular: ec2nodeclass scope: Cluster versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: EC2NodeClass is the Schema for the EC2NodeClass API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. - This will contain configuration necessary to launch instances in AWS. - properties: - amiFamily: - description: AMIFamily is the AMI family that instances use. - enum: - - AL2 - - AL2023 - - Bottlerocket - - Ubuntu - - Custom - - Windows2019 - - Windows2022 - type: string - amiSelectorTerms: - description: AMISelectorTerms is a list of or ami selector terms. - The terms are ORed. - items: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: description: |- - AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the ami id in EC2 - pattern: ami-[0-9a-z]+ - type: string - name: - description: |- - Name is the ami name in EC2. - This value is the name field, which is different from the name tag. - type: string - owner: - description: |- - Owner is the owner for the ami. - You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" - type: string - tags: - additionalProperties: + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 - type: object - x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') - type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: expected at least one, got none, ['tags', 'id', 'name'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) - - message: '''id'' is mutually exclusive, cannot be set with a combination - of other fields in amiSelectorTerms' - rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || - has(x.owner)))' - associatePublicIPAddress: - description: AssociatePublicIPAddress controls if public IP addresses - are assigned to instances that are launched with the nodeclass. - type: boolean - blockDeviceMappings: - description: BlockDeviceMappings to be applied to provisioned nodes. - items: - properties: - deviceName: - description: The device name (for example, /dev/sdh or xvdh). - type: string - ebs: - description: EBS contains parameters used to automatically set - up EBS volumes when an instance is launched. - properties: - deleteOnTermination: - description: DeleteOnTermination indicates whether the EBS - volume is deleted on instance termination. - type: boolean - encrypted: - description: |- - Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only - be attached to instances that support Amazon EBS encryption. If you are creating - a volume from a snapshot, you can't specify an encryption value. - type: boolean - iops: - description: |- - IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, - this represents the number of IOPS that are provisioned for the volume. For - gp2 volumes, this represents the baseline performance of the volume and the - rate at which the volume accumulates I/O credits for bursting. - - - The following are the supported values for each volume type: - - - * gp3: 3,000-16,000 IOPS - - - * io1: 100-64,000 IOPS - - - * io2: 100-64,000 IOPS - - - For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built - on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 IOPS. - - - This parameter is supported for io1, io2, and gp3 volumes only. This parameter - is not supported for gp2, st1, sc1, or standard volumes. - format: int64 - type: integer - kmsKeyID: - description: KMSKeyID (ARN) of the symmetric Key Management - Service (KMS) CMK used for encryption. - type: string - snapshotID: - description: SnapshotID is the ID of an EBS snapshot - type: string - throughput: - description: |- - Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. - Valid Range: Minimum value of 125. Maximum value of 1000. - format: int64 - type: integer - volumeSize: - description: |- - VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or - a volume size. The following are the supported volumes sizes for each volume - type: - - - * gp2 and gp3: 1-16,384 - - - * io1 and io2: 4-16,384 - - - * st1 and sc1: 125-16,384 - - - * standard: 1-1,024 - pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ - type: string - volumeType: - description: |- - VolumeType of the block device. - For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) - in the Amazon Elastic Compute Cloud User Guide. - enum: - - standard - - io1 - - io2 - - gp2 - - sc1 - - st1 - - gp3 + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: type: string - type: object - x-kubernetes-validations: - - message: snapshotID or volumeSize must be defined - rule: has(self.snapshotID) || has(self.volumeSize) - rootVolume: - description: |- - RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can - configure at most one root volume in BlockDeviceMappings. - type: boolean - type: object - maxItems: 50 - type: array - x-kubernetes-validations: - - message: must have only one blockDeviceMappings with rootVolume - rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() - <= 1 - context: - description: |- - Context is a Reserved field in EC2 APIs - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html - type: string - detailedMonitoring: - description: DetailedMonitoring controls if detailed monitoring is - enabled for instances that are launched - type: boolean - instanceProfile: - description: |- - InstanceProfile is the AWS entity that instances use. - This field is mutually exclusive from role. - The instance profile should already have a role assigned to it that Karpenter - has PassRole permission on for instance launch using this instanceProfile to succeed. - type: string - x-kubernetes-validations: - - message: instanceProfile cannot be empty - rule: self != '' - instanceStorePolicy: - description: InstanceStorePolicy specifies how to handle instance-store - disks. - enum: - - RAID0 - type: string - metadataOptions: - default: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: required - description: |- - MetadataOptions for the generated launch template of provisioned nodes. + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + The following are the supported values for each volume type: - This specifies the exposure of the Instance Metadata Service to - provisioned EC2 nodes. For more information, - see Instance Metadata and User Data - (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. + * gp3: 3,000-16,000 IOPS + * io1: 100-64,000 IOPS - Refer to recommended, security best practices - (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) - for limiting exposure of Instance Metadata and User Data to pods. - If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens - required. - properties: - httpEndpoint: - default: enabled - description: |- - HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned - nodes. If metadata options is non-nil, but this parameter is not specified, - the default state is "enabled". + * io2: 100-64,000 IOPS + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. - If you specify a value of "disabled", instance metadata will not be accessible - on the node. - enum: - - enabled - - disabled - type: string - httpProtocolIPv6: - default: disabled - description: |- - HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata - service on provisioned nodes. If metadata options is non-nil, but this parameter - is not specified, the default state is "disabled". - enum: - - enabled - - disabled - type: string - httpPutResponseHopLimit: - default: 2 - description: |- - HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for - instance metadata requests. The larger the number, the further instance - metadata requests can travel. Possible values are integers from 1 to 64. - If metadata options is non-nil, but this parameter is not specified, the - default value is 2. - format: int64 - maximum: 64 - minimum: 1 - type: integer - httpTokens: - default: required - description: |- - HTTPTokens determines the state of token usage for instance metadata - requests. If metadata options is non-nil, but this parameter is not - specified, the default state is "required". + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + * gp2 and gp3: 1-16,384 - If the state is optional, one can choose to retrieve instance metadata with - or without a signed token header on the request. If one retrieves the IAM - role credentials without a token, the version 1.0 role credentials are - returned. If one retrieves the IAM role credentials using a valid signed - token, the version 2.0 role credentials are returned. + * io1 and io2: 4-16,384 + * st1 and sc1: 125-16,384 - If the state is "required", one must send a signed token header with any - instance metadata retrieval requests. In this state, retrieving the IAM - role credentials always returns the version 2.0 credentials; the version - 1.0 credentials are not available. - enum: - - required - - optional - type: string - type: object - role: - description: |- - Role is the AWS identity that nodes use. This field is immutable. - This field is mutually exclusive from instanceProfile. - Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. - This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented - for the old instance profiles on an update. - type: string - x-kubernetes-validations: - - message: role cannot be empty - rule: self != '' - - message: immutable field changed - rule: self == oldSelf - securityGroupSelectorTerms: - description: SecurityGroupSelectorTerms is a list of or security group - selector terms. The terms are ORed. - items: + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: description: |- - SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. properties: - id: - description: ID is the security group id in EC2 - pattern: sg-[0-9a-z]+ - type: string - name: + clusterDNS: description: |- - Name is the security group name in EC2. - This value is the name field, which is different from the name tag. - type: string - tags: + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: additionalProperties: type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds type: object x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') - type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: securityGroupSelectorTerms cannot be empty - rule: self.size() != 0 - - message: expected at least one, got none, ['tags', 'id', 'name'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) - - message: '''id'' is mutually exclusive, cannot be set with a combination - of other fields in securityGroupSelectorTerms' - rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' - - message: '''name'' is mutually exclusive, cannot be set with a combination - of other fields in securityGroupSelectorTerms' - rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' - subnetSelectorTerms: - description: SubnetSelectorTerms is a list of or subnet selector terms. - The terms are ORed. - items: - description: |- - SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the subnet id in EC2 - pattern: subnet-[0-9a-z]+ - type: string - tags: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: additionalProperties: type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. type: object x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') - type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: subnetSelectorTerms cannot be empty - rule: self.size() != 0 - - message: expected at least one, got none, ['tags', 'id'] - rule: self.all(x, has(x.tags) || has(x.id)) - - message: '''id'' is mutually exclusive, cannot be set with a combination - of other fields in subnetSelectorTerms' - rule: '!self.all(x, has(x.id) && has(x.tags))' - tags: - additionalProperties: - type: string - description: Tags to be applied on ec2 resources like instances and - launch templates. - type: object - x-kubernetes-validations: - - message: empty tag keys aren't supported - rule: self.all(k, k != '') - - message: tag contains a restricted tag matching kubernetes.io/cluster/ - rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) - - message: tag contains a restricted tag matching karpenter.sh/nodepool - rule: self.all(k, k != 'karpenter.sh/nodepool') - - message: tag contains a restricted tag matching karpenter.sh/managed-by - rule: self.all(k, k !='karpenter.sh/managed-by') - - message: tag contains a restricted tag matching karpenter.sh/nodeclaim - rule: self.all(k, k !='karpenter.sh/nodeclaim') - - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass - rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') - userData: - description: |- - UserData to be applied to the provisioned nodes. - It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into - this UserData to ensure nodes are being provisioned with the correct configuration. - type: string - required: - - amiFamily - - securityGroupSelectorTerms - - subnetSelectorTerms - type: object - x-kubernetes-validations: - - message: amiSelectorTerms is required when amiFamily == 'Custom' - rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != - 0 : true' - - message: must specify exactly one of ['role', 'instanceProfile'] - rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) - && has(self.instanceProfile)) - - message: changing from 'instanceProfile' to 'role' is not supported. - You must delete and recreate this node class if you want to change - this. - rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) - && has(self.instanceProfile)) - status: - description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass - properties: - amis: - description: |- - AMI contains the current AMI values that are available to the - cluster under the AMI selectors. - items: - description: AMI contains resolved AMI selector values utilized - for node launch - properties: - id: - description: ID of the AMI - type: string - name: - description: Name of the AMI - type: string - requirements: - description: Requirements of the AMI to be utilized on an instance - type - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - required: - - id - - requirements + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) type: object - type: array - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional - helper methods + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. properties: - lastTransitionTime: + httpEndpoint: + default: enabled description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled type: string - message: + httpProtocolIPv6: + default: disabled description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string - observedGeneration: + httpPutResponseHopLimit: + default: 1 description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. format: int64 - minimum: 0 + maximum: 64 + minimum: 1 type: integer - reason: + httpTokens: + default: required description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. enum: - - "True" - - "False" - - Unknown + - required + - optional type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type type: object - type: array - instanceProfile: - description: InstanceProfile contains the resolved instance profile - for the role - type: string - securityGroups: - description: |- - SecurityGroups contains the current Security Groups values that are available to the - cluster under the SecurityGroups selectors. - items: - description: SecurityGroup contains resolved SecurityGroup selector - values utilized for node launch - properties: - id: - description: ID of the security group - type: string - name: - description: Name of the security group - type: string - required: - - id - type: object - type: array - subnets: - description: |- - Subnets contains the current Subnet values that are available to the - cluster under the subnet selectors. - items: - description: Subnet contains resolved Subnet selector values utilized - for node launch - properties: - id: - description: ID of the subnet - type: string - zone: - description: The associated availability zone - type: string - zoneID: - description: The associated availability zone ID - type: string - required: - - id - - zone + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + deprecated: + description: Deprecation status of the AMI + type: boolean + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current security group values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml b/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml index 19a213a..bfe259d 100644 --- a/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml +++ b/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.5 name: nodeclaims.karpenter.sh spec: group: karpenter.sh @@ -20,6 +20,9 @@ spec: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone name: Zone type: string @@ -32,8 +35,8 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - - jsonPath: .metadata.labels.karpenter\.sh/capacity-type - name: Capacity + - jsonPath: .status.providerID + name: ID priority: 1 type: string - jsonPath: .metadata.labels.karpenter\.sh/nodepool @@ -44,7 +47,7 @@ spec: name: NodeClass priority: 1 type: string - name: v1beta1 + name: v1 schema: openAPIV3Schema: description: NodeClaim is the Schema for the NodeClaims API @@ -69,133 +72,40 @@ spec: spec: description: NodeClaimSpec describes the desired state of the NodeClaim properties: - kubelet: + expireAfter: + default: 720h description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string nodeClassRef: description: NodeClassRef is a reference to an object that defines provider specific configuration properties: - apiVersion: + group: description: API version of the referent + pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: + - group + - kind - name type: object requirements: @@ -220,7 +130,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -257,7 +167,7 @@ spec: - key - operator type: object - maxItems: 30 + maxItems: 100 type: array x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined @@ -357,10 +267,30 @@ spec: - key type: object type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string required: - nodeClassRef - requirements type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf status: description: NodeClaimStatus defines the observed state of NodeClaim properties: @@ -425,12 +355,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -443,6 +368,13 @@ spec: imageID: description: ImageID is an identifier for the image that runs on the node type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string nodeName: description: NodeName is the name of the corresponding node object type: string diff --git a/charts/karpenter/crds/karpenter.sh_nodepools.yaml b/charts/karpenter/crds/karpenter.sh_nodepools.yaml index 20e2345..d53ecbb 100644 --- a/charts/karpenter/crds/karpenter.sh_nodepools.yaml +++ b/charts/karpenter/crds/karpenter.sh_nodepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.5 name: nodepools.karpenter.sh spec: group: karpenter.sh @@ -20,11 +20,28 @@ spec: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date - jsonPath: .spec.weight name: Weight priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 type: string - name: v1beta1 + name: v1 schema: openAPIV3Schema: description: NodePool is the Schema for the NodePools API @@ -55,8 +72,7 @@ spec: properties: disruption: default: - consolidationPolicy: WhenUnderutilized - expireAfter: 720h + consolidateAfter: 0s description: Disruption contains the parameters that relate to Karpenter's disruption logic properties: budgets: @@ -94,6 +110,19 @@ spec: Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array schedule: description: |- Schedule specifies when a budget begins being active, following @@ -118,29 +147,17 @@ spec: pattern: ^(([0-9]+(s|m|h))+)|(Never)$ type: string consolidationPolicy: - default: WhenUnderutilized + default: WhenEmptyOrUnderutilized description: |- ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation - algorithm. This policy defaults to "WhenUnderutilized" if not specified + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified enum: - WhenEmpty - - WhenUnderutilized - type: string - expireAfter: - default: 720h - description: |- - ExpireAfter is the duration the controller will wait - before terminating a node, measured from when the node is created. This - is useful to implement features like eventually consistent node upgrade, - memory leak protection, and disruption testing. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + - WhenEmptyOrUnderutilized type: string + required: + - consolidateAfter type: object - x-kubernetes-validations: - - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized - rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' - - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty - rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' limits: additionalProperties: anyOf: @@ -190,140 +207,55 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: - description: NodeClaimSpec describes the desired state of the NodeClaim + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. properties: - kubelet: + expireAfter: + default: 720h description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string nodeClassRef: description: NodeClassRef is a reference to an object that defines provider specific configuration properties: - apiVersion: + group: description: API version of the referent + pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: + - group + - kind - name type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind requirements: description: Requirements are layered with GetLabels and applied to every node. items: @@ -348,7 +280,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -385,7 +317,7 @@ spec: - key - operator type: object - maxItems: 30 + maxItems: 100 type: array x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined @@ -394,20 +326,6 @@ spec: rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - message: requirements with 'minValues' must have at least that many values specified in the 'values' field rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' - resources: - description: Resources models the resource requirements for the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources for the NodeClaim to launch - type: object - type: object - maxProperties: 0 startupTaints: description: |- StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically @@ -486,6 +404,23 @@ spec: - key type: object type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string required: - nodeClassRef - requirements @@ -509,6 +444,62 @@ spec: status: description: NodePoolStatus defines the observed state of NodePool properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array resources: additionalProperties: anyOf: diff --git a/charts/karpenter/templates/_helpers.tpl b/charts/karpenter/templates/_helpers.tpl index 9dce663..212f599 100644 --- a/charts/karpenter/templates/_helpers.tpl +++ b/charts/karpenter/templates/_helpers.tpl @@ -69,9 +69,9 @@ Karpenter image to use */}} {{- define "karpenter.controller.image" -}} {{- if .Values.controller.image.digest }} -{{- printf "%s:%s@%s" .Values.controller.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.controller.image.tag) .Values.controller.image.digest }} +{{- printf "%s:%s@%s" .Values.controller.image.repository (default (printf "%s" .Chart.AppVersion) .Values.controller.image.tag) .Values.controller.image.digest }} {{- else }} -{{- printf "%s:%s" .Values.controller.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.controller.image.tag) }} +{{- printf "%s:%s" .Values.controller.image.repository (default (printf "%s" .Chart.AppVersion) .Values.controller.image.tag) }} {{- end }} {{- end }} @@ -140,29 +140,3 @@ This works because Helm treats dictionaries as mutable objects and allows passin {{- include "karpenter.patchLabelSelector" (merge (dict "_target" $constraint) $) }} {{- end }} {{- end }} - -{{/* -Flatten the stdout logging outputs from args provided -*/}} -{{- define "karpenter.outputPathsList" -}} -{{ $paths := list -}} -{{- range .Values.logConfig.outputPaths -}} - {{- if not (has (printf "%s" . | quote) $paths) -}} - {{- $paths = printf "%s" . | quote | append $paths -}} - {{- end -}} -{{- end -}} -{{ $paths | join ", " }} -{{- end -}} - -{{/* -Flatten the stderr logging outputs from args provided -*/}} -{{- define "karpenter.errorOutputPathsList" -}} -{{ $paths := list -}} -{{- range .Values.logConfig.errorOutputPaths -}} - {{- if not (has (printf "%s" . | quote) $paths) -}} - {{- $paths = printf "%s" . | quote | append $paths -}} - {{- end -}} -{{- end -}} -{{ $paths | join ", " }} -{{- end -}} diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml index 1bce8bf..bc12a8a 100644 --- a/charts/karpenter/templates/clusterrole-core.yaml +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -36,19 +36,17 @@ rules: resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes"] + resources: ["storageclasses", "csinodes", "volumeattachments"] verbs: ["get", "watch", "list"] - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] verbs: ["list", "watch"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] - verbs: ["get", "watch", "list"] -{{- end }} - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", list, "watch"] # Write - apiGroups: ["karpenter.sh"] resources: ["nodeclaims", "nodeclaims/status"] @@ -61,16 +59,13 @@ rules: verbs: ["create", "patch"] - apiGroups: [""] resources: ["nodes"] - verbs: ["patch", "delete"] + verbs: ["patch", "delete", "update"] - apiGroups: [""] resources: ["pods/eviction"] verbs: ["create"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.sh", "validation.webhook.config.karpenter.sh"] -{{- end }} + - apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] {{- with .Values.additionalClusterRoleRules -}} {{ toYaml . | nindent 2 }} - {{- end -}} \ No newline at end of file + {{- end -}} diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml index a4c0856..769a343 100644 --- a/charts/karpenter/templates/clusterrole.yaml +++ b/charts/karpenter/templates/clusterrole.yaml @@ -35,14 +35,4 @@ rules: # Write - apiGroups: ["karpenter.k8s.aws"] resources: ["ec2nodeclasses", "ec2nodeclasses/status"] - verbs: ["patch", "update"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.k8s.aws"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["defaulting.webhook.karpenter.k8s.aws"] -{{- end }} + verbs: ["patch", "update"] \ No newline at end of file diff --git a/charts/karpenter/templates/configmap-logging.yaml b/charts/karpenter/templates/configmap-logging.yaml deleted file mode 100644 index d055fbc..0000000 --- a/charts/karpenter/templates/configmap-logging.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.logConfig.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: config-logging - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -data: - # https://github.com/uber-go/zap/blob/aa3e73ec0896f8b066ddf668597a02f89628ee50/config.go - zap-logger-config: | - { - "level": "{{ .Values.logConfig.logLevel.global }}", - "development": false, - "disableStacktrace": true, - "disableCaller": true, - "sampling": { - "initial": 100, - "thereafter": 100 - }, - "outputPaths": [{{ include "karpenter.outputPathsList" . }}], - "errorOutputPaths": [{{ include "karpenter.errorOutputPathsList" . }}], - "encoding": "{{ .Values.logConfig.logEncoding }}", - "encoderConfig": { - "timeKey": "time", - "levelKey": "level", - "nameKey": "logger", - "callerKey": "caller", - "messageKey": "message", - "stacktraceKey": "stacktrace", - "levelEncoder": "capital", - "timeEncoder": "iso8601" - } - } - loglevel.controller: {{ .Values.logConfig.logLevel.controller }} - loglevel.webhook: {{ .Values.logConfig.logLevel.webhook }} -{{- end }} \ No newline at end of file diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml index d1994b3..990ce48 100644 --- a/charts/karpenter/templates/deployment.yaml +++ b/charts/karpenter/templates/deployment.yaml @@ -56,8 +56,11 @@ spec: {{- if .Values.hostNetwork }} hostNetwork: true {{- end }} + {{- with .Values.schedulerName }} + schedulerName: {{ . | quote }} + {{- end }} containers: - - name: controller + - name: {{ .Values.controller.containerName | default "controller" }} securityContext: runAsUser: 65532 runAsGroup: 65532 @@ -76,17 +79,17 @@ spec: value: "1.19.0-0" - name: KARPENTER_SERVICE value: {{ include "karpenter.fullname" . }} - {{- if .Values.webhook.enabled }} - - name: WEBHOOK_PORT - value: "{{ .Values.webhook.port }}" - - name: WEBHOOK_METRICS_PORT - value: "{{ .Values.webhook.metrics.port }}" - - name: DISABLE_WEBHOOK - value: "false" - {{- end }} {{- with .Values.logLevel }} - name: LOG_LEVEL value: "{{ . }}" + {{- end }} + {{- with .Values.logOutputPaths }} + - name: LOG_OUTPUT_PATHS + value: "{{ join "," . }}" + {{- end }} + {{- with .Values.logErrorOutputPaths }} + - name: LOG_ERROR_OUTPUT_PATHS + value: "{{ join "," . }}" {{- end }} - name: METRICS_PORT value: "{{ .Values.controller.metrics.port }}" @@ -103,7 +106,7 @@ spec: divisor: "0" resource: limits.memory - name: FEATURE_GATES - value: "Drift={{ .Values.settings.featureGates.drift }},SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }}" + value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }},NodeRepair={{ .Values.settings.featureGates.nodeRepair }}" {{- with .Values.settings.batchMaxDuration }} - name: BATCH_MAX_DURATION value: "{{ . }}" @@ -112,14 +115,6 @@ spec: - name: BATCH_IDLE_DURATION value: "{{ . }}" {{- end }} - {{- with .Values.settings.assumeRoleARN }} - - name: ASSUME_ROLE_ARN - value: "{{ . }}" - {{- end }} - {{- with .Values.settings.assumeRoleDuration }} - - name: ASSUME_ROLE_DURATION - value: "{{ . }}" - {{- end }} {{- with .Values.settings.clusterCABundle }} - name: CLUSTER_CA_BUNDLE value: "{{ . }}" @@ -136,6 +131,10 @@ spec: - name: ISOLATED_VPC value: "{{ . }}" {{- end }} + {{- with .Values.settings.eksControlPlane }} + - name: EKS_CONTROL_PLANE + value: "{{ . }}" + {{- end }} {{- with .Values.settings.vmMemoryOverheadPercent }} - name: VM_MEMORY_OVERHEAD_PERCENT value: "{{ . }}" @@ -159,14 +158,6 @@ spec: - name: http-metrics containerPort: {{ .Values.controller.metrics.port }} protocol: TCP - {{- if .Values.webhook.enabled }} - - name: webhook-metrics - containerPort: {{ .Values.webhook.metrics.port }} - protocol: TCP - - name: https-webhook - containerPort: {{ .Values.webhook.port }} - protocol: TCP - {{- end }} - name: http containerPort: {{ .Values.controller.healthProbe.port }} protocol: TCP @@ -186,12 +177,8 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or (.Values.logConfig.enabled) (.Values.controller.extraVolumeMounts) }} + {{- if .Values.controller.extraVolumeMounts }} volumeMounts: - {{- if .Values.logConfig.enabled }} - - name: config-logging - mountPath: /etc/karpenter/logging - {{- end }} {{- with .Values.controller.extraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -228,13 +215,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- if or (.Values.logConfig.enabled) (.Values.extraVolumes) }} + {{- if .Values.extraVolumes }} volumes: - {{- if .Values.logConfig.enabled }} - - name: config-logging - configMap: - name: config-logging - {{- end }} {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/karpenter/templates/role.yaml b/charts/karpenter/templates/role.yaml index 3862e5b..2be0918 100644 --- a/charts/karpenter/templates/role.yaml +++ b/charts/karpenter/templates/role.yaml @@ -14,19 +14,7 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch"] -{{- if .Values.webhook.enabled }} - - apiGroups: [""] - resources: ["configmaps", "secrets"] - verbs: ["get", "list", "watch"] -{{- end }} # Write -{{- if .Values.webhook.enabled }} - - apiGroups: [""] - resources: ["secrets"] - verbs: ["update"] - resourceNames: - - "{{ include "karpenter.fullname" . }}-cert" -{{- end }} - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["patch", "update"] @@ -55,24 +43,3 @@ rules: resources: ["services"] resourceNames: ["kube-dns"] verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "karpenter.fullname" . }}-lease - namespace: kube-node-lease - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -rules: - # Read - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch"] - # Write - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["delete"] \ No newline at end of file diff --git a/charts/karpenter/templates/rolebinding.yaml b/charts/karpenter/templates/rolebinding.yaml index 8606a49..ba54e18 100644 --- a/charts/karpenter/templates/rolebinding.yaml +++ b/charts/karpenter/templates/rolebinding.yaml @@ -33,26 +33,6 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: {{ include "karpenter.fullname" . }}-dns -subjects: - - kind: ServiceAccount - name: {{ template "karpenter.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "karpenter.fullname" . }}-lease - namespace: kube-node-lease - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "karpenter.fullname" . }}-lease subjects: - kind: ServiceAccount name: {{ template "karpenter.serviceAccountName" . }} diff --git a/charts/karpenter/templates/secret-webhook-cert.yaml b/charts/karpenter/templates/secret-webhook-cert.yaml deleted file mode 100644 index b17309f..0000000 --- a/charts/karpenter/templates/secret-webhook-cert.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if .Values.webhook.enabled }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "karpenter.fullname" . }}-cert - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -# data: {} # Injected by karpenter-webhook -{{- end }} diff --git a/charts/karpenter/templates/service.yaml b/charts/karpenter/templates/service.yaml index 2c880b7..694265c 100644 --- a/charts/karpenter/templates/service.yaml +++ b/charts/karpenter/templates/service.yaml @@ -5,10 +5,15 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} + {{- if or .Values.additionalAnnotations .Values.service.annotations }} annotations: + {{- with .Values.additionalAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: type: ClusterIP ports: @@ -16,15 +21,5 @@ spec: port: {{ .Values.controller.metrics.port }} targetPort: http-metrics protocol: TCP - {{- if .Values.webhook.enabled }} - - name: webhook-metrics - port: {{ .Values.webhook.metrics.port }} - targetPort: webhook-metrics - protocol: TCP - - name: https-webhook - port: {{ .Values.webhook.port }} - targetPort: https-webhook - protocol: TCP - {{- end }} selector: {{- include "karpenter.selectorLabels" . | nindent 4 }} diff --git a/charts/karpenter/templates/webhooks-core.yaml b/charts/karpenter/templates/webhooks-core.yaml deleted file mode 100644 index 55297a1..0000000 --- a/charts/karpenter/templates/webhooks-core.yaml +++ /dev/null @@ -1,69 +0,0 @@ -{{- if .Values.webhook.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.karpenter.sh - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - nodeclaims - - nodeclaims/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - nodepools - - nodepools/status - scope: '*' ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.config.karpenter.sh - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.config.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - objectSelector: - matchLabels: - app.kubernetes.io/part-of: {{ template "karpenter.name" . }} -{{- end }} \ No newline at end of file diff --git a/charts/karpenter/templates/webhooks.yaml b/charts/karpenter/templates/webhooks.yaml deleted file mode 100644 index 4815743..0000000 --- a/charts/karpenter/templates/webhooks.yaml +++ /dev/null @@ -1,67 +0,0 @@ -{{- if .Values.webhook.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: defaulting.webhook.karpenter.k8s.aws - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: defaulting.webhook.karpenter.k8s.aws - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - ec2nodeclasses - - ec2nodeclasses/status - scope: '*' ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.karpenter.k8s.aws - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.karpenter.k8s.aws - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - ec2nodeclasses - - ec2nodeclasses/status - scope: '*' -{{- end }} \ No newline at end of file diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index d032d87..a1d16eb 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -12,6 +12,9 @@ additionalAnnotations: {} imagePullPolicy: IfNotPresent # -- Image pull secrets for Docker images. imagePullSecrets: [] +service: + # -- Additional annotations for the Service. + annotations: {} serviceAccount: # -- Specifies if a ServiceAccount should be created. create: true @@ -27,8 +30,8 @@ serviceMonitor: enabled: false # -- Additional labels for the ServiceMonitor. additionalLabels: {} - # -- Configuration on `http-metrics` endpoint for the ServiceMonitor. - # Not to be used to add additional endpoints. + # -- Configuration on `http-metrics` endpoint for the ServiceMonitor. + # Not to be used to add additional endpoints. # See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint endpointConfig: {} # -- Number of replicas. @@ -56,6 +59,8 @@ terminationGracePeriodSeconds: # -- Bind the pod to the host network. # This is required when using a custom CNI. hostNetwork: false +# -- Specify which Kubernetes scheduler should dispatch the pod. +schedulerName: default-scheduler # -- Configure the DNS Policy for the pod dnsPolicy: ClusterFirst # -- Configure DNS Config for the pod @@ -81,7 +86,7 @@ affinity: topologySpreadConstraints: - maxSkew: 1 topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway + whenUnsatisfiable: DoNotSchedule # -- Tolerations to allow the pod to be scheduled to nodes with taints. tolerations: - key: CriticalAddonsOnly @@ -97,13 +102,15 @@ extraVolumes: [] # expirationSeconds: 86400 # path: token controller: + # -- Distinguishing container name (containerName: karpenter-controller). + containerName: controller image: # -- Repository path to the controller image. repository: public.ecr.aws/karpenter/controller # -- Tag of the controller image. - tag: 0.37.0 + tag: 1.1.0 # -- SHA256 digest of the controller image. - digest: sha256:157f478f5db1fe999f5e2d27badcc742bf51cc470508b3cebe78224d0947674f + digest: sha256:51bca600197c7c6e6e0838549664b2c12c3f8dd4b23744ab28202ae97ca5aed1 # -- Additional environment variables for the controller pod. env: [] # - name: AWS_REGION @@ -133,40 +140,18 @@ controller: sidecarVolumeMounts: [] metrics: # -- The container port to use for metrics. - port: 8000 + port: 8080 healthProbe: # -- The container port to use for http health probe. port: 8081 -webhook: - # -- Whether to enable the webhooks and webhook permissions. - enabled: false - # -- The container port to use for the webhook. - port: 8443 - metrics: - # -- The container port to use for webhook metrics. - port: 8001 # -- Global log level, defaults to 'info' logLevel: info -# -- Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) -logConfig: - # -- Whether to enable provisioning and mounting the log ConfigMap - enabled: false - # -- Log outputPaths - defaults to stdout only - outputPaths: - - stdout - # -- Log errorOutputPaths - defaults to stderr only - errorOutputPaths: - - stderr - # -- Log encoding - defaults to json - must be one of 'json', 'console' - logEncoding: json - # -- Component-based log configuration - logLevel: - # -- Global log level, defaults to 'info' - global: info - # -- Controller log level, defaults to 'info' - controller: info - # -- Error log level, defaults to 'error' - webhook: error +# -- Log outputPaths - defaults to stdout only +logOutputPaths: + - stdout +# -- Log errorOutputPaths - defaults to stderr only +logErrorOutputPaths: + - stderr # -- Global Settings to configure Karpenter settings: # -- The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one @@ -176,10 +161,6 @@ settings: # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods # will be batched separately. batchIdleDuration: 1s - # -- Role to assume for calling AWS services. - assumeRoleARN: "" - # -- Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless assumeRoleARN set. - assumeRoleDuration: 15m # -- Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. clusterCABundle: "" # -- Cluster name. @@ -189,7 +170,9 @@ settings: # -- If true then assume we can't reach AWS services which don't have a VPC endpoint # This also has the effect of disabling look-ups to the AWS pricing endpoint isolatedVPC: false - # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types + # Marking this true means that your cluster is running with an EKS control plane and Karpenter should attempt to discover cluster details from the DescribeCluster API + eksControlPlane: false + # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. The value of `0.075` equals to 7.5%. vmMemoryOverheadPercent: 0.075 # -- Interruption queue is the name of the SQS queue used for processing interruption events from EC2 # Interruption handling is disabled if not specified. Enabling interruption handling may @@ -201,10 +184,9 @@ settings: # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features featureGates: - # -- drift is in BETA and is enabled by default. - # Setting drift to false disables the drift disruption method to watch for drift between currently deployed nodes - # and the desired state of nodes set in nodepools and nodeclasses - drift: true # -- spotToSpotConsolidation is ALPHA and is disabled by default. # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. spotToSpotConsolidation: false + # -- nodeRepair is ALPHA and is disabled by default. + # Setting this to true will enable node repair. + nodeRepair: false