From d61247ed8ca06a87e2ad8e71d21bce34b6a80815 Mon Sep 17 00:00:00 2001 From: thorner Date: Tue, 28 Apr 2020 15:30:10 -0400 Subject: [PATCH] - Change the default storage class in the deployment manifest to linode-block-storage-retain. - Change the storage class in the example PVC to linode-block-storage-retain. - Update README to reflect these changes. --- README.md | 15 +- ...linode-blockstorage-csi-driver-v0.1.5.yaml | 547 ++++++++++++++++++ .../examples/kubernetes/csi-pvc.yaml | 2 +- 3 files changed, 556 insertions(+), 8 deletions(-) create mode 100644 pkg/linode-bs/deploy/releases/linode-blockstorage-csi-driver-v0.1.5.yaml diff --git a/README.md b/README.md index 00cf82cb..20bd87cc 100644 --- a/README.md +++ b/README.md @@ -62,26 +62,27 @@ linode Opaque 2 18h The following command will deploy the CSI driver with the related Kubernetes volume attachment, driver registration, and provisioning sidecars: ```sh -kubectl apply -f https://raw.githubusercontent.com/linode/linode-blockstorage-csi-driver/master/pkg/linode-bs/deploy/releases/linode-blockstorage-csi-driver-v0.1.4.yaml +kubectl apply -f https://raw.githubusercontent.com/linode/linode-blockstorage-csi-driver/master/pkg/linode-bs/deploy/releases/linode-blockstorage-csi-driver-v0.1.5.yaml ``` This deployment is a concatenation of all of the `yaml` files in [pkg/linode-bs/deploy/kubernetes/](https://github.com/linode/linode-blockstorage-csi-driver/tree/master/pkg/linode-bs/deploy/kubernetes/). Notably, this deployment will: -* set the default storage class to `linode-block-storage` [Learn More](https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/) +* set the default storage class to `linode-block-storage-retain` [Learn More](https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/) This behavior can be modified in the [csi-storageclass.yaml](https://github.com/linode/linode-blockstorage-csi-driver/blob/master/pkg/linode-bs/deploy/kubernetes/05-csi-storageclass.yaml) section of the deployment by toggling the `storageclass.kubernetes.io/is-default-class` annotation. ```sh $ kubectl get storageclasses NAME PROVISIONER AGE - linode-block-storage (default) linodebs.csi.linode.com 2d + linode-block-storage-retain (default) linodebs.csi.linode.com 2d + linode-block-storage linodebs.csi.linode.com 2d ``` -* use a `reclaimPolicy` of `Released` [Learn More](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/) +* use a `reclaimPolicy` of `Retain` [Learn More](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/) - Volumes created by this CSI driver will automatically be deleted when they are no longer needed. + Volumes created by this CSI driver will default to using the `linode-block-storage-retain` storage class if one is not specified. Upon deletion of all PersitentVolumeClaims, the PersistentVolume and its backing Block Storage Volume will remain intact. * assume that the [Linode CCM](https://github.com/linode/linode-cloud-controller-manager) is initialized and running [Learn More](https://kubernetes.io/docs/reference/command-line-tools-reference/cloud-controller-manager/) @@ -89,9 +90,9 @@ Notably, this deployment will: ### Example Usage -This repository contains [two manifests](https://github.com/linode/linode-blockstorage-csi-driver/tree/master/pkg/linode-bs/examples/kubernetes) that demonstrate use of the Linode BlockStorage CSI. These manifests will create a PersistentVolume Claim using the `linode-block-storage` storage class and then consume it in a minimal pod. +This repository contains [two manifests](https://github.com/linode/linode-blockstorage-csi-driver/tree/master/pkg/linode-bs/examples/kubernetes) that demonstrate use of the Linode BlockStorage CSI. These manifests will create a PersistentVolume Claim using the `linode-block-storage-retain` storage class and then consume it in a minimal pod. -Once you have installed the Linode BlockStorage CSI, the following commands will run the example. Be sure to delete the pod and PVC and ensure that the Linode BlockStorage Volume has been deleted. +Once you have installed the Linode BlockStorage CSI, the following commands will run the example. Once you are finished with the example, please be sure to delete the pod, PVC, and the associated Block Storage Volume. The PVC created in this example uses the `linode-block-storage-retain` storage class, so you will need to remove the Block Storage Volume from your Linode account via the Cloud Manager or the Linode CLI. ```sh kubectl create -f https://raw.githubusercontent.com/linode/linode-blockstorage-csi-driver/master/pkg/linode-bs/examples/kubernetes/csi-pvc.yaml diff --git a/pkg/linode-bs/deploy/releases/linode-blockstorage-csi-driver-v0.1.5.yaml b/pkg/linode-bs/deploy/releases/linode-blockstorage-csi-driver-v0.1.5.yaml new file mode 100644 index 00000000..6cbce2db --- /dev/null +++ b/pkg/linode-bs/deploy/releases/linode-blockstorage-csi-driver-v0.1.5.yaml @@ -0,0 +1,547 @@ +# pkg/linode-bs/deploy/kubernetes/01-csi-nodeinfo.yaml +# Requires CSINodeInfo feature gate (alpha in 1.12) +# xref: https://raw.githubusercontent.com/kubernetes/csi-api/master/pkg/crd/manifests/csidriver.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: csinodeinfos.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: csi.storage.k8s.io + version: v1alpha1 + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their specs. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + status: + description: Status of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their statuses. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + available: + description: Whether the CSI driver is installed. + type: boolean + volumePluginMechanism: + description: Indicates to external components the required mechanism + to use for any in-tree plugins replaced by this driver. + pattern: in-tree|csi + type: string +--- +# pkg/linode-bs/deploy/kubernetes/02-csi-driver.yaml +# Requires CSIDriverRegistry feature gate (alpha in 1.12) +# xref: https://raw.githubusercontent.com/kubernetes/csi-api/master/pkg/crd/manifests/csinodeinfo.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: csidrivers.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + version: v1alpha1 + group: csi.storage.k8s.io + names: + kind: CSIDriver + plural: csidrivers + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of the CSI Driver. + properties: + attachRequired: + description: Indicates this CSI volume driver requires an attach operation, + and that Kubernetes should call attach and wait for any attach operation + to complete before proceeding to mount. + type: boolean + podInfoOnMountVersion: + description: Indicates this CSI volume driver requires additional pod + information (like podName, podUID, etc.) during mount operations. + type: string +--- +# pkg/linode-bs/deploy/kubernetes/03-accounts-roles-bindings.yaml +##### Node Service Account, Roles, RoleBindings +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-node-sa + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: driver-registrar-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: driver-registrar-binding + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: driver-registrar-role + apiGroup: rbac.authorization.k8s.io +--- +##### Controller Service Account, Roles, Rolebindings +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-controller-sa + namespace: kube-system +--- +# xref: https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-provisioner-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-provisioner-binding + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: external-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +# xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-attacher-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-attacher-binding + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: external-attacher-role + apiGroup: rbac.authorization.k8s.io +--- +# xref: https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: external-snapshotter-role + namespace: kube-system +rules: +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] +- apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["watch", "get", "list"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-snapshotter-binding + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: external-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- +# pkg/linode-bs/deploy/kubernetes/04-csi-driver-instance.yaml +apiVersion: csi.storage.k8s.io/v1alpha1 +kind: CSIDriver +metadata: + name: linodebs.csi.linode.com +spec: + attachRequired: true + podInfoOnMountVersion: "v1" +--- +# pkg/linode-bs/deploy/kubernetes/05-csi-storageclass.yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: linode-block-storage + namespace: kube-system +provisioner: linodebs.csi.linode.com +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: linode-block-storage-retain + namespace: kube-system + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: linodebs.csi.linode.com +reclaimPolicy: Retain +--- +# pkg/linode-bs/deploy/kubernetes/06-ss-csi-linode-controller.yaml +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-linode-controller + namespace: kube-system +spec: + serviceName: "csi-linode" + replicas: 1 + selector: + matchLabels: + app: csi-linode-controller + template: + metadata: + labels: + app: csi-linode-controller + role: csi-linode + spec: + serviceAccount: csi-controller-sa + initContainers: + - name: init + image: bitnami/kubectl:1.16.3-debian-10-r36 + command: + - /scripts/get-linode-id.sh + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: linode-info + mountPath: /linode-info + - name: get-linode-id + mountPath: /scripts + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.0.0 + args: + - "--volume-name-prefix=pvc" + - "--volume-name-uuid-length=16" + - "--csi-address=$(ADDRESS)" + - "--v=2" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.0 + args: + - "--v=2" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: linode-csi-plugin + image: linode/linode-blockstorage-csi-driver:v0.1.4 + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--token=$(LINODE_TOKEN)" + - "--url=$(LINODE_API_URL)" + - "--node=$(NODE_NAME)" + - "--bs-prefix=$(LINODE_BS_PREFIX)" + - "--v=2" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: LINODE_API_URL + value: https://api.linode.com/v4 + - name: LINODE_BS_PREFIX + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LINODE_TOKEN + valueFrom: + secretKeyRef: + name: linode + key: token + imagePullPolicy: "Always" + volumeMounts: + - name: linode-info + mountPath: /linode-info + - name: get-linode-id + mountPath: /scripts + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} + - name: linode-info + emptyDir: {} + - name: get-linode-id + configMap: + name: get-linode-id + # octal mode 755 + defaultMode: 493 +--- +# pkg/linode-bs/deploy/kubernetes/07-ds-csi-linode-node.yaml +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-linode-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-linode-node + template: + metadata: + labels: + app: csi-linode-node + role: csi-linode + spec: + serviceAccount: csi-node-sa + initContainers: + - name: init + image: bitnami/kubectl:1.16.3-debian-10-r36 + command: + - /scripts/get-linode-id.sh + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: linode-info + mountPath: /linode-info + - name: get-linode-id + mountPath: /scripts + hostNetwork: true + containers: + - name: driver-registrar + image: quay.io/k8scsi/driver-registrar:v1.0-canary + args: + - "--v=2" + - "--csi-address=$(ADDRESS)" + - "--mode=node-register" + - "--driver-requires-attachment=true" + - "--pod-info-mount-version=\"v1\"" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/linodebs.csi.linode.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: csi-linode-plugin + image: linode/linode-blockstorage-csi-driver:v0.1.4 + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--token=$(LINODE_TOKEN)" + - "--url=$(LINODE_API_URL)" + - "--node=$(NODE_NAME)" + - "--v=2" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: LINODE_API_URL + value: https://api.linode.com/v4 + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LINODE_TOKEN + valueFrom: + secretKeyRef: + name: linode + key: token + imagePullPolicy: "Always" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: linode-info + mountPath: /linode-info + - name: get-linode-id + mountPath: /scripts + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + - mountPath: /dev + name: device-dir + volumes: + - name: linode-info + emptyDir: {} + - name: get-linode-id + configMap: + name: get-linode-id + defaultMode: 493 + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/linodebs.csi.linode.com + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + # The following mounts are required to trigger host udevadm from container + - name: udev-rules-etc + hostPath: + path: /etc/udev + type: Directory + - name: udev-rules-lib + hostPath: + path: /lib/udev + type: Directory + - name: udev-socket + hostPath: + path: /run/udev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory +--- +# pkg/linode-bs/deploy/kubernetes/08-cm-get-linode-id.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: get-linode-id + namespace: kube-system + labels: + app: csi-linode +data: + get-linode-id.sh: |- + #!/bin/bash -efu + id="$(kubectl get node/"${NODE_NAME}" -o jsonpath='{.spec.providerID}')" + if [[ ! -z "${id}" ]]; then + echo "${id}" + echo -n "${id:9}" > /linode-info/linode-id + exit 0 + fi + echo "Provider ID not found" + # Exit here so that we wait for the CCM to initialize the provider ID + exit 1 diff --git a/pkg/linode-bs/examples/kubernetes/csi-pvc.yaml b/pkg/linode-bs/examples/kubernetes/csi-pvc.yaml index f66d64da..bcdf309c 100644 --- a/pkg/linode-bs/examples/kubernetes/csi-pvc.yaml +++ b/pkg/linode-bs/examples/kubernetes/csi-pvc.yaml @@ -8,4 +8,4 @@ spec: resources: requests: storage: 10Gi - storageClassName: linode-block-storage + storageClassName: linode-block-storage-retain