diff --git a/README.md b/README.md index 3f661ab..3dd1ae2 100755 --- a/README.md +++ b/README.md @@ -1,52 +1,71 @@ -## Ansible Playbooks to install an air gapped Highly Available Kubernetes cluster -Uses Kubeadm with the --experimental-control-plane switch to install the Stacked Control Plane. This deployment pattern is well suited to single purpose, secure, infrastructure software clusters with low frequency container changes. +# Ansible Playbooks to install an HA Kubernetes (multi-master) cluster using Kubeadm. -- Uses recently GA'd Kubeadm 1.13 HA multi-master joining features -- Uses the Ansible Host as a short lived forward proxy to download cluster software and container images -- No container registry is required (downloaded images are cache to the local docker engines) +This repository provides Ansible Playbooks to install a Kubernetes HA cluster in an airgapped environment. +- Uses recently GA'd Kubeadm HA joining features -### Prerequisites: +# Prerequisites: +- Install Ansible and a forward proxy on the Ansible host + - Ansible: + for macos `brew install ansible` + for linux `yum install ansible` -* Setup ssh access from Ansible host to Kubernetes nodes. +- Setup ssh access from Ansible host to Kubernetes nodes. ```ssh-copy-id -i ~/.ssh/id_rsa.pub ``` -- Setup a local forward proxy with Internet access on the Ansible host (e.g. tinyproxy). Update the proxy environment variable details in `inventory/group_vars/all`. -![Air Gapped Network Flows](./airgapped-ansible-kubernetes.png) +# Environment preparation: +Specify the Master and Workers in the `inventory/*cluster*` file: +``` +[k8s-masters] # these are all the masters +[k8s-workers] # these are all the worker nodes +``` -### Environment preparation: - -Specify the Master and Worker hostnames in the `inventory/cluster` file: - -Update the `inventory/group_vars` sections: +Update the `inventory/group_vars/*cluster*` section: - choose the desired versions for kubernetes and docker - setup the pod network cidr (default setup is for calico - modify in calico.yaml as well) - specify the version of Helm to use - specify the Local Storage Provisioner version -### Install process: +# Install a highly available kubernetes using kubeadm + +You can now run install-all.yaml playbook to get your cluster setup. +You can also run the different playbooks separately for different purposes (setting up docker, masters, kubeadm, heml ...). + +``` +ansible-playbook -i inventory/cluster1-prod playbooks/install-all.yaml --private-key=~/.ssh/id_rsa -u %username% -v +``` + +# Restarting the install: +If you need to restart the process using kubeadm reset, please use the uninstall.yaml playbook that deletes the state from all vms. + -Run install-all.yaml playbook to get your cluster fully setup. Note port 8888 below is the default for tinyproxy- adjust according to your proxy install. -You can also run the different playbooks separately for different purposes (setting up docker, masters, kubeadm, helm ...). +# Upgrade a highly available kubernetes using kubeadm +To upgrade the kubernetes control plane run: ``` -ansible-playbook -i inventory/cluster playbooks/install-all.yaml --private-key=~/.ssh/id_rsa -u username --extra-vars ansible_ssh_extra_args="-R8888:localhost:8888" -v +ansible-playbook -i inventory/cluster1-prod playbooks/upgrade-all.yaml --private-key=~/.ssh/id_rsa -u username -v ``` -### What install-all.yaml includes: - -- Adds the required yum repositories -- Installs docker -- Installs kubeadm, kubelet and kubectl -- Initializes the first master with etcd and kubernetes-api -- Installs Calico networking -- Joins replica master nodes to the primary master -- Adds the worker nodes to the cluster -- Installs Helm & Tiller -- Install Kubernetes Dashboard (Metrics Server support coming 02/19) -- Installs Local Storage Provisioner (usefull for cloud native, shared nothing, stateful set apps such as Kafka, Zookeeper, Elastic) - -### Still to do: -- Update Replica Master /etc/kubernetes/* file permissions after Ansible copy \ No newline at end of file +# What install-all.yaml includes: + +- Adding the required yum repositories +- Installing docker +- Installing kubeadm, kubelet and kubectl +- Initializing the first master with etcd and kubernetes-api +- Join replica master nodes to the primary master +- Adding the worker nodes to the cluster +- Installing Helm & Tiller +- Installing Local Storage Provisioner +- Enable Azure AD OIDC authentication + +# Restarting the install: + +If you need to restart the process using kubeadm reset, please use the uninstall.yaml playbook that deletes the state from all vms. + +# To sequentially drain and patch the underlying OS hosts: + +``` +ansible-playbook -i inventory/cluster1-prod playbooks/os-patch-updates.yaml --private-key=~/.ssh/id_rsa -u username -v +``` \ No newline at end of file diff --git a/airgapped-ansible-kubernetes.png b/airgapped-ansible-kubernetes.png deleted file mode 100644 index acf834d..0000000 Binary files a/airgapped-ansible-kubernetes.png and /dev/null differ diff --git a/ansible.cfg b/ansible.cfg index 99292ba..efd2bfe 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -5,3 +5,8 @@ inventory = inventory roles_path = roles display_skipped_hosts = false any_errors_fatal = true + +#[ssh_connection] +#pipelining = True +#control_path = /tmp/ansible-ssh-%%h-%%p-%%r +#ssh_args = -C -o ControlMaster=auto -o ControlPersist=1800 diff --git a/inventory/cluster b/inventory/cluster1-prod old mode 100755 new mode 100644 similarity index 50% rename from inventory/cluster rename to inventory/cluster1-prod index ab18c86..4dbe335 --- a/inventory/cluster +++ b/inventory/cluster1-prod @@ -1,19 +1,22 @@ [k8s-master-primary] -master1 +master1.domain.local [k8s-master-replicas] -master2 -master3 +master2.domain.local +master3.domain.local [k8s-masters:children] k8s-master-primary k8s-master-replicas [k8s-workers] -worker1 -worker2 -worker3 +worker1.domain.local +worker2.domain.local +worker3.domain.local [k8s-nodes:children] k8s-masters k8s-workers + +[cluster1-prod:children] +k8s-nodes diff --git a/inventory/cluster2-nonprod b/inventory/cluster2-nonprod new file mode 100644 index 0000000..8a96d6f --- /dev/null +++ b/inventory/cluster2-nonprod @@ -0,0 +1,22 @@ +[k8s-master-primary] +master1.domain.local + +[k8s-master-replicas] +master2.domain.local +master3.domain.local + +[k8s-masters:children] +k8s-master-primary +k8s-master-replicas + +[k8s-workers] +worker1.domain.local +worker2.domain.local +worker3.domain.local + +[k8s-nodes:children] +k8s-masters +k8s-workers + +[cluster2-nonprod:children] +k8s-nodes diff --git a/inventory/group_vars/all b/inventory/group_vars/all deleted file mode 100644 index 62aed68..0000000 --- a/inventory/group_vars/all +++ /dev/null @@ -1,41 +0,0 @@ ---- - -docker_version: 18.09.1 -kubernetes_version: v1.13.1 -kubelet_version: 1.13.1 - -# proxy server to tunnel through SSH to ansible deployment host -proxy_env: - http_proxy: http://localhost:8888 - https_proxy: http://localhost:8888 - no_proxy: 127.0.0.1,10.0.0.0 - -# kubernetes API load balanced VIP for HA installations -kubernetes_loadbalanced_api_dns: k8sapi01.domain.local - -# Docker Daemon configuration -docker_ce_daemon_options: - exec-opts: [ "native.cgroupdriver=systemd" ] - log-driver: json-file - log-opts: - max-size: "100m" - max-file: "7" - storage-driver: overlay2 - storage-opts: [ "overlay2.override_kernel_check=true" ] - -# Kubernetes Kubeadm Cluster Configuration -kubeadm_config_options: - apiVersion: kubeadm.k8s.io/v1beta1 - kind: ClusterConfiguration - kubernetesVersion: "{{ kubernetes_version }}" - apiServer: - certSANs: - - "{{ kubernetes_loadbalanced_api_dns }}" - controlPlaneEndpoint: "{{ kubernetes_loadbalanced_api_dns }}:6443" - networking: - podSubnet: 10.244.0.0/16 - -# Addon Container Images -tiller_image: gcr.io/kubernetes-helm/tiller:v2.12.1 -local_volume_provisioner_image: quay.io/external_storage/local-volume-provisioner:v2.2.0 -kubernetes_dashboard_image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 diff --git a/inventory/group_vars/cluster1-prod/all b/inventory/group_vars/cluster1-prod/all new file mode 100644 index 0000000..f089d5d --- /dev/null +++ b/inventory/group_vars/cluster1-prod/all @@ -0,0 +1,58 @@ +--- + +docker_version: 18.09.6 +kubernetes_version: v1.15.0 +kubeadm_version: 1.15.0 + +# kubernetes API load balanced VIP for HA installations +kubernetes_loadbalanced_api_dns: k8s-api01.domain.local +kubernetes_cluster_label: cluster1-prod + +# Container registry01 +container_registry: container-registry01.nonprod.domain.local + +# Docker Daemon configuration +docker_ce_daemon_options: + exec-opts: [ "native.cgroupdriver=systemd" ] + log-driver: json-file + log-opts: + max-size: "100m" + max-file: "7" + storage-driver: overlay2 + storage-opts: [ "overlay2.override_kernel_check=true" ] + +# Kubernetes Kubeadm Cluster Configuration +kubeadm_config_options: + apiVersion: kubeadm.k8s.io/v1beta1 + kind: ClusterConfiguration + kubernetesVersion: "{{ kubernetes_version }}" + apiServer: + certSANs: + - "{{ kubernetes_loadbalanced_api_dns }}" + controlPlaneEndpoint: "{{ kubernetes_loadbalanced_api_dns }}:6443" + networking: + podSubnet: 10.244.0.0/16 + imageRepository: "{{ container_registry }}/keystone" + useHyperKubeImage: true + clusterName: "{{ kubernetes_cluster_label }}" + etcd: + local: + imageRepository: "{{ container_registry }}/keystone" + + +# Addon Container Images +tiller_image: "{{ container_registry }}/keystone/tiller:v2.12.1" + +# Filebeat Kafka Topic Name +filebeat_app_id: kubernetes-logs-prod + +# Metricbeat Kafka Topic Name +metricbeat_app_id: metricbeat-kubernetes-prod + +# Azure AD Admin group and AppID +aad_admin_groupname: XXXXX +aad_admin_groupid: XXXXX + +# Azure AD User group and AppID +aad_user_groupname: XXXXX +aad_user_groupid: XXXXX \ No newline at end of file diff --git a/inventory/group_vars/cluster2-nonprod/all b/inventory/group_vars/cluster2-nonprod/all new file mode 100644 index 0000000..d96b562 --- /dev/null +++ b/inventory/group_vars/cluster2-nonprod/all @@ -0,0 +1,58 @@ +--- + +docker_version: 18.09.6 +kubernetes_version: v1.15.0 +kubeadm_version: 1.15.0 + +# kubernetes API load balanced VIP for HA installations +kubernetes_loadbalanced_api_dns: k8s-api02.nonprod.domain.local +kubernetes_cluster_label: cluster2-nonprod + +# Container registry01 +container_registry: container-registry01.nonprod.domain.local + +# Docker Daemon configuration +docker_ce_daemon_options: + exec-opts: [ "native.cgroupdriver=systemd" ] + log-driver: json-file + log-opts: + max-size: "100m" + max-file: "7" + storage-driver: overlay2 + storage-opts: [ "overlay2.override_kernel_check=true" ] + +# Kubernetes Kubeadm Cluster Configuration +kubeadm_config_options: + apiVersion: kubeadm.k8s.io/v1beta1 + kind: ClusterConfiguration + kubernetesVersion: "{{ kubernetes_version }}" + apiServer: + certSANs: + - "{{ kubernetes_loadbalanced_api_dns }}" + controlPlaneEndpoint: "{{ kubernetes_loadbalanced_api_dns }}:6443" + networking: + podSubnet: 10.244.0.0/16 + imageRepository: "{{ container_registry }}/keystone" + useHyperKubeImage: true + clusterName: "{{ kubernetes_cluster_label }}" + etcd: + local: + imageRepository: "{{ container_registry }}/keystone" + + +# Addon Container Images +tiller_image: "{{ container_registry }}/keystone/tiller:v2.12.1" + +# Filebeat Kafka Topic Name +filebeat_app_id: kubernetes-logs-nonprod + +# Metricbeat Kafka Topic Name +metricbeat_app_id: metricbeat-kubernetes-nonprod + +# Azure AD Admin group and AppID +aad_admin_groupname: XXXXX +aad_admin_groupid: XXXXX + +# Azure AD User group and AppID +aad_user_groupname: XXXXX +aad_user_groupid: XXXXX \ No newline at end of file diff --git a/playbooks/aad-authentication.yaml b/playbooks/aad-authentication.yaml new file mode 100755 index 0000000..68edfd9 --- /dev/null +++ b/playbooks/aad-authentication.yaml @@ -0,0 +1,4 @@ +- hosts: k8s-master-primary + become: yes + roles: + - aad-authentication \ No newline at end of file diff --git a/playbooks/kafka-charts.yaml b/playbooks/elastic-search.yaml similarity index 71% rename from playbooks/kafka-charts.yaml rename to playbooks/elastic-search.yaml index 0358dd6..1eb37e5 100755 --- a/playbooks/kafka-charts.yaml +++ b/playbooks/elastic-search.yaml @@ -1,4 +1,4 @@ - hosts: k8s-master-primary become: yes roles: - - kafka-charts + - elastic-search \ No newline at end of file diff --git a/playbooks/install-all.yaml b/playbooks/install-all.yaml index 4ab22a1..b250a6a 100755 --- a/playbooks/install-all.yaml +++ b/playbooks/install-all.yaml @@ -2,10 +2,13 @@ - import_playbook: repos.yaml - import_playbook: docker.yaml - import_playbook: kubeadm-prep.yaml +#- import_playbook: kubeadm-singlenode-cluster.yaml - import_playbook: kubeadm-init-master.yaml - import_playbook: calico.yaml - import_playbook: kubeadm-join-masters.yaml - import_playbook: kubeadm-join-workers.yaml - import_playbook: helm-install.yaml - import_playbook: local-storage-provisioner.yaml -- import_playbook: kubernetes-dashboard.yaml \ No newline at end of file +- import_playbook: kubernetes-dashboard.yaml +- import_playbook: elastic-search.yaml +- import_playbook: aad-authentication.yaml \ No newline at end of file diff --git a/playbooks/install_docker.yaml b/playbooks/install_docker.yaml new file mode 100755 index 0000000..50b959d --- /dev/null +++ b/playbooks/install_docker.yaml @@ -0,0 +1,29 @@ +- name: Install Docker-CE engine + yum: + name: docker-ce-{{ docker_version }} + state: installed + enablerepo: docker-ce + +- name: Create directory for trusted registry + file: + path: /etc/docker/certs.d/container-registry01.nonprod.domain.local + state: directory + +- name: Copy registry CA certificate + template: + src: container-registry01.nonprod.domain.local-ca.crt + dest: /etc/docker/certs.d/container-registry01.nonprod.domain.local/ca.crt + force: yes + +- name: Configure additional engine options + copy: + content: "{{ docker_ce_daemon_options | to_nice_json }}" + dest: /etc/docker/daemon.json + mode: 0644 + when: docker_ce_daemon_options is defined + +- name: Enable docker service + systemd: + name: docker + state: restarted + enabled: yes \ No newline at end of file diff --git a/playbooks/kubeadm-singlenode-cluster.yaml b/playbooks/kubeadm-singlenode-cluster.yaml new file mode 100755 index 0000000..0e26e61 --- /dev/null +++ b/playbooks/kubeadm-singlenode-cluster.yaml @@ -0,0 +1,4 @@ +- hosts: k8s-master-primary + become: yes + roles: + - kubeadm-singlenode-cluster \ No newline at end of file diff --git a/playbooks/kubeadm-upgrade-master.yaml b/playbooks/kubeadm-upgrade-master.yaml new file mode 100755 index 0000000..eac6d7d --- /dev/null +++ b/playbooks/kubeadm-upgrade-master.yaml @@ -0,0 +1,4 @@ +- hosts: k8s-master-primary + become: yes + roles: + - kubeadm-upgrade-masters \ No newline at end of file diff --git a/playbooks/kubeadm-upgrade-workers.yaml b/playbooks/kubeadm-upgrade-workers.yaml new file mode 100755 index 0000000..a512ec0 --- /dev/null +++ b/playbooks/kubeadm-upgrade-workers.yaml @@ -0,0 +1,5 @@ +- hosts: k8s-workers + become: yes + serial: 1 + roles: + - kubeadm-upgrade-workers diff --git a/playbooks/nginx-install.yaml b/playbooks/nginx-install.yaml new file mode 100755 index 0000000..4a3fe85 --- /dev/null +++ b/playbooks/nginx-install.yaml @@ -0,0 +1,4 @@ +- hosts: k8s-master-primary + become: yes + roles: + - ingress-nginx diff --git a/playbooks/os-patch-updates.yaml b/playbooks/os-patch-updates.yaml new file mode 100755 index 0000000..743305e --- /dev/null +++ b/playbooks/os-patch-updates.yaml @@ -0,0 +1,4 @@ +- hosts: k8s-masters:k8s-workers + become: yes + roles: + - os-patch-updates \ No newline at end of file diff --git a/playbooks/upgrade-all.yaml b/playbooks/upgrade-all.yaml new file mode 100755 index 0000000..61008a2 --- /dev/null +++ b/playbooks/upgrade-all.yaml @@ -0,0 +1,5 @@ +#- import_playbook: execute-backup.yaml +- import_playbook: kubeadm-upgrade-master.yaml +#- import_playbook: calico.yaml +#- import_playbook: kubeadm-upgrade-masterreplicas.yaml +- import_playbook: kubeadm-upgrade-workers.yaml diff --git a/playbooks/upgrade_worker.yaml b/playbooks/upgrade_worker.yaml new file mode 100755 index 0000000..9f4998d --- /dev/null +++ b/playbooks/upgrade_worker.yaml @@ -0,0 +1,35 @@ + - name: Upgrade kubenetes packages + yum: + name: "{{ packages }}" + enablerepo: kubernetes + vars: + packages: + - kubelet-{{kubeadm_version}} + - kubectl-{{kubeadm_version}} + - kubeadm-{{kubeadm_version}} + + - name: Drain kubernetes worker node + shell: kubectl drain {{ inventory_hostname_short }} --delete-local-data --ignore-daemonsets + register: kubeadm_drain + delegate_to: "{{ groups['k8s-masters'][0] }}" + + - pause: + prompt: "Wait for drain" + + - name: Upgrade worker nodes + shell: kubeadm upgrade node config --kubelet-version {{kubernetes_version}} + + - name: Restart kubelet + systemd: + state: restarted + daemon_reload: yes + name: kubelet + enabled: yes + + - name: Uncordon worker node + shell: kubectl uncordon {{ inventory_hostname_short }} + register: kubeadm_uncordon + delegate_to: "{{ groups['k8s-masters'][0] }}" + + - pause: + prompt: "Wait for uncordon" \ No newline at end of file diff --git a/roles/aad-authentication/README.md b/roles/aad-authentication/README.md new file mode 100644 index 0000000..70e2275 --- /dev/null +++ b/roles/aad-authentication/README.md @@ -0,0 +1,88 @@ + +## Notes on using Azure AD as an authentication provider for Kubernetes Adminstration + +https://github.com/kubernetes/client-go/tree/master/plugin/pkg/client/auth/azure?source=post_page--------------------------- + +`On the "Manage" / "Manifest" set groupMembershipClaims property to SecurityGroup` + +Azure Tenant ID: XXXXX + +AzureAD API Server SP: ts-container-platform-sp +appid: XXXXX + +AzureAD User Auth SP: +appid: XXXXX + +### 1. Allow cluster firewall access to https://sts.windows.net from k8s nodes + +### 2. Set up kubernetes API-server with Azure AD ts-container-platform-sp SP (Applied by Ansible) + +Update /etc/kubernetes/manifests/kube-apiserver.yaml (Applied by Ansible) + +``` +--oidc-client-id=spn:XXXXX +--oidc-issuer-url=https://sts.windows.net/XXXXX/ +--oidc-username-claim=upn +--oidc-groups-claim=groups +``` + +### 3. Set Azure Group access for the cluster + +#### Admin NonProduction Cluster Role +``` +cat << EOF | kubectl apply -f - +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: XXXXX:cluster-admin +subjects: + - kind: Group + name: XXXXX + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +EOF +``` + +#### User NonProduction Cluster Role +``` +cat << EOF | kubectl apply -f - +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: XXXXX:edit +subjects: + - kind: Group + name: XXXXX + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: edit + apiGroup: rbac.authorization.k8s.io +EOF +``` + + + +### 4. Update client side kubectl config with azure user and cluster configuration + +``` +kubectl config set-credentials azureuser@domain.local \ +--auth-provider=azure \ +--auth-provider-arg=environment=AzurePublicCloud \ +--auth-provider-arg=client-id=XXXXX \ +--auth-provider-arg=tenant-id=XXXXX \ +--auth-provider-arg=apiserver-id=XXXXX + +kubectl config set-cluster cluster1-nonprod \ +--server=https://k8s-api01.nonprod.domain.local:6443 \ +--insecure-skip-tls-verify=true + +kubectl config set-context cluster1-nonprod_azureuser@domain.local \ +--cluster=cluster1-nonprod \ +--user=azureuser@domain.local + +kubectl config use-context cluster1-nonprod_azureuser@domain.local +``` \ No newline at end of file diff --git a/roles/aad-authentication/tasks/aad_masters.yaml b/roles/aad-authentication/tasks/aad_masters.yaml new file mode 100644 index 0000000..2146f05 --- /dev/null +++ b/roles/aad-authentication/tasks/aad_masters.yaml @@ -0,0 +1,9 @@ +- name: Add Azure OIDC authentication to Kubernetes + blockinfile: + path: /etc/kubernetes/manifests/kube-apiserver.yaml + insertafter: '^ - kube-apiserver' + block: |2 + - --oidc-client-id=spn:XXXXX + - --oidc-issuer-url=https://sts.windows.net/XXXXX/ + - --oidc-username-claim=upn + - --oidc-groups-claim=groups \ No newline at end of file diff --git a/roles/aad-authentication/tasks/main.yaml b/roles/aad-authentication/tasks/main.yaml new file mode 100644 index 0000000..11eaad0 --- /dev/null +++ b/roles/aad-authentication/tasks/main.yaml @@ -0,0 +1,43 @@ + +- name: Upgrade masters sequentially + include_tasks: "aad_masters.yaml" + with_items: "{{ groups['k8s-masters'] }}" + when: "hostvars[host_item].inventory_hostname == inventory_hostname" + loop_control: + loop_var: host_item + +- name: Create AAD Admin ClusterRoleBindings + shell: + cmd: | + cat << EOF | kubectl apply -f - + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: {{ aad_admin_groupname }}:cluster-admin + subjects: + - kind: Group + name: {{ aad_admin_groupid }} + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + EOF + +- name: Create AAD User ClusterRoleBindings + shell: + cmd: | + cat << EOF | kubectl apply -f - + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: {{ aad_user_groupname }}:cluster-admin + subjects: + - kind: Group + name: {{ aad_user_groupid }} + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + EOF \ No newline at end of file diff --git a/roles/backup-cronjob/README.md b/roles/backup-cronjob/README.md new file mode 100644 index 0000000..43d9f79 --- /dev/null +++ b/roles/backup-cronjob/README.md @@ -0,0 +1,16 @@ +# CronJob to backup ETCD state to S3 storage + +### 2. Update API yaml environment variables + +For example: +- S3_FOLDER=k8skaf-api05-prod +- S3_BUCKET=prod-container-registry +- cron schedule + +### 3. Upload Kubernetes API and S3 secrets to kubernetes + +Create kubernetes secrets for the UCP password and S3 access keys +``` +kubectl create -n kube-system -f ./etcd-backup-cronjob.yaml +kubectl create -n kube-system -f ./s3-storage-secret.yaml +``` \ No newline at end of file diff --git a/roles/backup-cronjob/tasks/main.yaml b/roles/backup-cronjob/tasks/main.yaml new file mode 100644 index 0000000..f22fe5c --- /dev/null +++ b/roles/backup-cronjob/tasks/main.yaml @@ -0,0 +1,24 @@ + +- name: Copy s3 storage secret + template: + src: s3-storage-secret.yaml + dest: /tmp/s3-storage-secret.yaml + force: yes + +- name: Create S3 storage secret + shell: kubectl apply -f /tmp/s3-storage-secret.yaml + +- name: Copy etcd-backup manifests + template: + src: etcd-backup-cronjob.yaml + dest: /tmp/etcd-backup-cronjob.yaml + force: yes + +- name: Set S3 Folder Name within template + replace: + path: /tmp/etcd-backup-cronjob.yaml + regexp: "kubernetes_cluster_label" + replace: "{{ kubernetes_cluster_label }}" + +- name: Create Etcd Backup Cronjob Deployments + shell: kubectl apply -f /tmp/etcd-backup-cronjob.yaml \ No newline at end of file diff --git a/roles/backup-cronjob/templates/etcd-backup-cronjob.yaml b/roles/backup-cronjob/templates/etcd-backup-cronjob.yaml new file mode 100644 index 0000000..5e04589 --- /dev/null +++ b/roles/backup-cronjob/templates/etcd-backup-cronjob.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: etcd-backup + namespace: kube-system +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 7 + jobTemplate: + spec: + template: + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: "Exists" + hostNetwork: true + containers: + - name: etcd-backup + image: container-registry01.nonprod.domain.local/system/etcd-backup:v1 + imagePullPolicy: IfNotPresent + command: ['sh', '-c', 'etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key snapshot save /tmp/etcd-snapshot-$(date +%Y-%m-%d_%H:%M:%S_%Z).db && s3cmd --access_key=$ACCESS_KEY --secret_key=$SECRET_KEY --host=ashobjectstorage.domain.local --host-bucket=$S3_BUCKET.ashobjectstorage.domain.local --no-check-certificate put /tmp/etcd-snapshot-$(date +%Y-%m-%d_%H:%M:%S_%Z).db s3://$S3_BUCKET/backups/$S3_FOLDER/'] + securityContext: + allowPrivilegeEscalation: true + env: + - name: ETCDCTL_API + value: "3" + - name: S3_BUCKET + value: "nonprod-container-registry" + - name: S3_FOLDER + value: "kubernetes_cluster_label" + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-storage-secret + key: access_key + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: s3-storage-secret + key: secret_key + volumeMounts: + - name: etcd-certs + mountPath: /etc/kubernetes/pki/etcd + readOnly: true + - mountPath: /tmp + name: tmp + volumes: + - name: etcd-certs + hostPath: + path: /etc/kubernetes/pki/etcd + type: DirectoryOrCreate + - name: tmp + hostPath: + path: /tmp + type: DirectoryOrCreate + restartPolicy: OnFailure + terminationGracePeriodSeconds: 45 + securityContext: + runAsNonRoot: false + runAsUser: 0 + schedule: '00 00 * * 6' + successfulJobsHistoryLimit: 7 + suspend: false \ No newline at end of file diff --git a/roles/calico/README.md b/roles/calico/README.md new file mode 100644 index 0000000..6a37758 --- /dev/null +++ b/roles/calico/README.md @@ -0,0 +1,2 @@ + +curl https://docs.projectcalico.org/v3.8/manifests/calico.yaml -O diff --git a/roles/calico/tasks/main.yaml b/roles/calico/tasks/main.yaml index 060ccfb..8d31c32 100755 --- a/roles/calico/tasks/main.yaml +++ b/roles/calico/tasks/main.yaml @@ -1,15 +1,14 @@ -- name: Copy Calico RBAC rbac-kdd.yaml manifests - template: - src: rbac-kdd.yaml - dest: /tmp/rbac-kdd.yaml - -- name: Create Calico RBAC Deployment - shell: kubectl apply -f /tmp/rbac-kdd.yaml - - name: Copy calico.yaml manifests template: src: calico.yaml dest: /tmp/calico.yaml + force: yes + +- name: Set Calico container image value + replace: + path: /tmp/calico.yaml + regexp: "image: calico" + replace: "image: {{ container_registry }}/keystone" - name: Create Calico Deployment shell: kubectl apply -f /tmp/calico.yaml diff --git a/roles/calico/templates/calico.yaml b/roles/calico/templates/calico.yaml index 89408c7..2b51ffe 100644 --- a/roles/calico/templates/calico.yaml +++ b/roles/calico/templates/calico.yaml @@ -1,9 +1,5 @@ -# Calico Version v3.3.2 -# https://docs.projectcalico.org/v3.3/releases#v3.3.2 -# This manifest includes the following component versions: -# calico/node:v3.3.2 -# calico/cni:v3.3.2 - +--- +# Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap apiVersion: v1 @@ -11,11 +7,9 @@ metadata: name: calico-config namespace: kube-system data: - # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas - # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is - # essential. + # Typha is disabled. typha_service_name: "none" - # Configure the Calico backend to use. + # Configure the backend to use. calico_backend: "bird" # Configure the MTU to use @@ -26,7 +20,7 @@ data: cni_network_config: |- { "name": "k8s-pod-network", - "cniVersion": "0.3.0", + "cniVersion": "0.3.1", "plugins": [ { "type": "calico", @@ -35,8 +29,7 @@ data: "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { - "type": "host-local", - "subnet": "usePodCidr" + "type": "calico-ipam" }, "policy": { "type": "k8s" @@ -54,138 +47,428 @@ data: } --- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMBlock + plural: ipamblocks + singular: ipamblock -# This manifest creates a Service, which will be backed by Calico's Typha daemon. -# Typha sits in between Felix and the API server, reducing Calico's load on the API server. +--- -apiVersion: v1 -kind: Service +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha + name: blockaffinities.crd.projectcalico.org spec: - ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha - selector: - k8s-app: calico-typha + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BlockAffinity + plural: blockaffinities + singular: blockaffinity --- -# This manifest creates a Deployment of Typha to back the above service. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMHandle + plural: ipamhandles + singular: ipamhandle -apiVersion: apps/v1beta1 -kind: Deployment +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha + name: ipamconfigs.crd.projectcalico.org spec: - # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the - # typha_service_name variable in the calico-config ConfigMap above. - # - # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential - # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In - # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 - revisionHistoryLimit: 2 - template: - metadata: - labels: - k8s-app: calico-typha - annotations: - # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical - # add-on, ensuring it gets priority scheduling and that its resources are reserved - # if it ever gets evicted. - scheduler.alpha.kubernetes.io/critical-pod: '' - cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' - spec: - nodeSelector: - beta.kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - # Since Calico can't network a pod until Typha is up, we need to run Typha itself - # as a host-networked pod. - serviceAccountName: calico-node - containers: - - image: quay.io/calico/typha:v3.3.2 - name: calico-typha - ports: - - containerPort: 5473 - name: calico-typha - protocol: TCP - env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" - livenessProbe: - exec: - command: - - calico-typha - - check - - liveness - periodSeconds: 30 - initialDelaySeconds: 30 - readinessProbe: - exec: - command: - - calico-typha - - check - - readiness - periodSeconds: 10 + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMConfig + plural: ipamconfigs + singular: ipamconfig --- -# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer + +--- -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha + name: bgpconfigurations.crd.projectcalico.org spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-typha + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration --- -# This manifest installs the calico/node container, as well -# as the Calico CNI plugins and network config on +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkSet + plural: networksets + singular: networkset +--- +# Source: calico/templates/rbac.yaml + +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only requried for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system + +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 metadata: name: calico-node namespace: kube-system @@ -226,22 +509,80 @@ spec: # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: calico/cni:v3.8.1 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.8.1 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: calico/pod2daemon-flexvol:v3.8.1 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver containers: - # Runs calico/node container on each Kubernetes node. This + # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v3.3.2 + image: calico/node:v3.8.1 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" @@ -275,7 +616,7 @@ spec: # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. - name: CALICO_IPV4POOL_CIDR - value: "10.244.0.0/16" + value: "192.168.0.0/16" # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -323,39 +664,10 @@ spec: - mountPath: /var/lib/calico name: var-lib-calico readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: quay.io/calico/cni:v3.3.2 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir + - name: policysync + mountPath: /var/run/nodeagent volumes: - # Used by calico/node. + # Used by calico-node. - name: lib-modules hostPath: path: /lib/modules @@ -376,6 +688,22 @@ spec: - name: cni-net-dir hostPath: path: /etc/cni/net.d + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds --- apiVersion: v1 @@ -385,139 +713,72 @@ metadata: namespace: kube-system --- +# Source: calico/templates/calico-kube-controllers.yaml -# Create all the CustomResourceDefinitions needed for -# Calico policy and networking mode. - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: FelixConfiguration - plural: felixconfigurations - singular: felixconfiguration ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPPeer - plural: bgppeers - singular: bgppeer - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPConfiguration - plural: bgpconfigurations - singular: bgpconfiguration - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPPool - plural: ippools - singular: ippool - ---- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment metadata: - name: hostendpoints.crd.projectcalico.org + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: HostEndpoint - plural: hostendpoints - singular: hostendpoint + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: calico/kube-controllers:v3.8.1 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition +apiVersion: v1 +kind: ServiceAccount metadata: - name: clusterinformations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: ClusterInformation - plural: clusterinformations - singular: clusterinformation - + name: calico-kube-controllers + namespace: kube-system --- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkPolicy - plural: globalnetworkpolicies - singular: globalnetworkpolicy +# Source: calico/templates/calico-etcd-secrets.yaml --- - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkSet - plural: globalnetworksets - singular: globalnetworkset +# Source: calico/templates/calico-typha.yaml --- +# Source: calico/templates/configure-canal.yaml -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkPolicy - plural: networkpolicies - singular: networkpolicy diff --git a/roles/calico/templates/rbac-kdd.yaml b/roles/calico/templates/rbac-kdd.yaml deleted file mode 100644 index 11fa50d..0000000 --- a/roles/calico/templates/rbac-kdd.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# Calico Version v3.3.2 -# https://docs.projectcalico.org/v3.3/releases#v3.3.2 -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node -rules: - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system diff --git a/roles/docker/tasks/main.yaml b/roles/docker/tasks/main.yaml index f371dc8..22df5f0 100755 --- a/roles/docker/tasks/main.yaml +++ b/roles/docker/tasks/main.yaml @@ -5,36 +5,10 @@ with_items: - device-mapper-persistent-data - lvm2 - -- name: Install Docker-CE engine - yum: - name: docker-ce-{{ docker_version }} - state: installed -- name: Create /etc/docker/ folder - file: - path: "/etc/docker/ " - state: directory - owner: "root" - -- name: Configure additional engine options - copy: - content: "{{ docker_ce_daemon_options | to_nice_json }}" - dest: /etc/docker/daemon.json - mode: 0644 - when: docker_ce_daemon_options is defined - -- name: Create Docker HTTP Proxy config file - blockinfile: - path: /etc/systemd/system/docker.service.d/http_proxy.conf - create: yes - block: | - [Service] - Environment="HTTPS_PROXY=http://127.0.0.1:8888/" "NO_PROXY=localhost,127.0.0.1,10.0.0.0" - -- name: restart docker - systemd: - name: docker - state: restarted - daemon_reload: yes - enabled: yes \ No newline at end of file +- name: Install docker on hosts sequentially + include_tasks: "install_docker.yaml" + with_items: "{{ groups['k8s-nodes'] }}" + when: "hostvars[host_item].inventory_hostname == inventory_hostname" + loop_control: + loop_var: host_item diff --git a/roles/docker/templates/container-registry01.nonprod.domain.local-ca.crt b/roles/docker/templates/container-registry01.nonprod.domain.local-ca.crt new file mode 100644 index 0000000..23ee432 --- /dev/null +++ b/roles/docker/templates/container-registry01.nonprod.domain.local-ca.crt @@ -0,0 +1,3 @@ +-----BEGIN CERTIFICATE----- +XXXXX +-----END CERTIFICATE----- diff --git a/roles/elastic-search/tasks/main.yaml b/roles/elastic-search/tasks/main.yaml new file mode 100644 index 0000000..8ff766e --- /dev/null +++ b/roles/elastic-search/tasks/main.yaml @@ -0,0 +1,68 @@ +- name: copy kube-state-metrics manifests + template: + src: kube-state-metrics.yaml + dest: /tmp/kube-state-metrics.yaml + force: yes + +- name: Set Kube State Metrics container image value + replace: + path: /tmp/kube-state-metrics.yaml + regexp: "image: quay.io/coreos" + replace: "image: {{ container_registry }}/system" + +- name: Set Kube State Metrics container image value + replace: + path: /tmp/kube-state-metrics.yaml + regexp: "image: k8s.gcr.io" + replace: "image: {{ container_registry }}/system" + +- name: Create Kube State Metrics Deployments + shell: kubectl apply -f /tmp/kube-state-metrics.yaml + + +- name: copy filebeat manifests + template: + src: filebeat-kubernetes.yaml + dest: /tmp/filebeat-kubernetes.yaml + force: yes + +- name: Set filebeat container image value + replace: + path: /tmp/filebeat-kubernetes.yaml + regexp: "image: docker.elastic.co/beats" + replace: "image: {{ container_registry }}/system" + +- name: Set filebeat app_id value + replace: + path: /tmp/filebeat-kubernetes.yaml + regexp: "app_id: filebeat_app_id" + replace: "app_id: {{ filebeat_app_id }}" + +- name: Create filebeat Deployments + shell: kubectl apply -f /tmp/filebeat-kubernetes.yaml + +- name: copy metricbeat manifests + template: + src: metricbeat-kubernetes.yaml + dest: /tmp/metricbeat-kubernetes.yaml + force: yes + +- name: Set metricbeat container image value + replace: + path: /tmp/metricbeat-kubernetes.yaml + regexp: "image: docker.elastic.co/beats" + replace: "image: {{ container_registry }}/system" + +- name: Set metricbeat app_id value + replace: + path: /tmp/metricbeat-kubernetes.yaml + regexp: "app_id: metricbeat_app_id" + replace: "app_id: {{ metricbeat_app_id }}" + +- name: Create metricbeat Deployments + shell: kubectl apply -f /tmp/metricbeat-kubernetes.yaml + +- name: Label kubernetes nodes with cluster tag + shell: kubectl label node {{ hostvars[item].inventory_hostname_short }} "cluster={{kubernetes_cluster_label}}" + delegate_to: "{{ groups['k8s-master-primary'][0] }}" + with_items: "{{ groups['k8s-nodes'] }}" diff --git a/roles/elastic-search/templates/filebeat-kubernetes.yaml b/roles/elastic-search/templates/filebeat-kubernetes.yaml new file mode 100644 index 0000000..d105c5e --- /dev/null +++ b/roles/elastic-search/templates/filebeat-kubernetes.yaml @@ -0,0 +1,181 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat +data: + filebeat.yml: |- + filebeat.config: + #inputs: + # Mounted `filebeat-inputs` configmap: + # path: ${path.config}/inputs.d/*.yml + # Reload inputs configs as they change: + # reload.enabled: false + modules: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this: + filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + labels.dedot: true + annotations.dedot: true + + processors: + - add_cloud_metadata: + - add_kubernetes_metadata: + labels.dedot: true + annotations.dedot: true + # - drop_fields: # label added by helm by default and causes failure in Beats. This can be moved with Beats 6.7 release which has fix + # when: + # has_fields: ['kubernetes.labels.app'] + # fields: + # - 'kubernetes.labels.app' + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.kafka: + # initial brokers for reading cluster metadata + hosts: ["kafrck-vccn010.domain.local:31090", "kafrck-vccn011.domain.local:31091", "kafrck-vccn012.domain.local:31092", "kafrck-vccn013.domain.local:31093", "kafrck-vccn014.domain.local:31094"] + # message topic selection + partitioning + topic: '%{[fields.app_id]}' + partition.round_robin: + reachable_only: false + required_acks: 1 + compression: gzip + max_message_bytes: 1000000 + + # Optional fields that you can specify to add additional information to the + # output. + fields: + app_id: filebeat_app_id +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-inputs + namespace: kube-system + labels: + k8s-app: filebeat +data: + kubernetes.yml: |- + - type: docker + containers.ids: + - "*" + processors: + - add_kubernetes_metadata: + in_cluster: true +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:6.7.1 + imagePullPolicy: IfNotPresent + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 768Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: inputs + mountPath: /usr/share/filebeat/inputs.d + readOnly: true + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: inputs + configMap: + defaultMode: 0600 + name: filebeat-inputs + # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart + - name: data + hostPath: + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- \ No newline at end of file diff --git a/roles/elastic-search/templates/kube-state-metrics.yaml b/roles/elastic-search/templates/kube-state-metrics.yaml new file mode 100644 index 0000000..2008563 --- /dev/null +++ b/roles/elastic-search/templates/kube-state-metrics.yaml @@ -0,0 +1,181 @@ +apiVersion: apps/v1 +# Kubernetes versions after 1.9.0 should use apps/v1 +# Kubernetes versions before 1.8.0 should use apps/v1beta1 or extensions/v1beta1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-state-metrics + replicas: 1 + template: + metadata: + labels: + k8s-app: kube-state-metrics + spec: + serviceAccountName: kube-state-metrics + containers: + - name: kube-state-metrics + imagePullPolicy: IfNotPresent + image: quay.io/coreos/kube-state-metrics:v1.5.0 + ports: + - name: http-metrics + containerPort: 8080 + - name: telemetry + containerPort: 8081 + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + - name: addon-resizer + imagePullPolicy: IfNotPresent + image: k8s.gcr.io/addon-resizer:1.8.3 + resources: + limits: + cpu: 150m + memory: 50Mi + requests: + cpu: 150m + memory: 50Mi + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - /pod_nanny + - --container=kube-state-metrics + - --cpu=200m + - --extra-cpu=1m + - --memory=200Mi + - --extra-memory=2Mi + - --threshold=5 + - --deployment=kube-state-metrics +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kube-state-metrics +rules: +- apiGroups: [""] + resources: + - configmaps + - secrets + - nodes + - pods + - services + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: ["list", "watch"] +- apiGroups: ["extensions"] + resources: + - daemonsets + - deployments + - replicasets + verbs: ["list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +- apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: ["list", "watch"] +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: kube-state-metrics + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kube-state-metrics-resizer +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + namespace: kube-system + name: kube-state-metrics-resizer +rules: +- apiGroups: [""] + resources: + - pods + verbs: ["get"] +- apiGroups: ["extensions"] + resources: + - deployments + resourceNames: ["kube-state-metrics"] + verbs: ["get", "update"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-state-metrics + namespace: kube-system + labels: + k8s-app: kube-state-metrics + annotations: + prometheus.io/scrape: 'true' +spec: + ports: + - name: http-metrics + port: 8080 + targetPort: http-metrics + protocol: TCP + - name: telemetry + port: 8081 + targetPort: telemetry + protocol: TCP + selector: + k8s-app: kube-state-metrics +--- \ No newline at end of file diff --git a/roles/elastic-search/templates/metricbeat-kubernetes.yaml b/roles/elastic-search/templates/metricbeat-kubernetes.yaml new file mode 100644 index 0000000..87fae9c --- /dev/null +++ b/roles/elastic-search/templates/metricbeat-kubernetes.yaml @@ -0,0 +1,371 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + # To enable hints based autodiscover uncomment this: + metricbeat.autodiscover: + providers: + - type: kubernetes + host: ${NODE_NAME} + hints.enabled: true + labels.dedot: true + annotations.dedot: true + + processors: + - add_cloud_metadata: + - add_kubernetes_metadata: + labels.dedot: true + annotations.dedot: true + - drop_fields: + when: + has_fields: ['kubernetes.labels.app'] + fields: + - 'kubernetes.labels.app' + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.kafka: + # initial brokers for reading cluster metadata + hosts: ["kafrck-vccn010.domain.local:31090", "kafrck-vccn011.domain.local:31091", "kafrck-vccn012.domain.local:31092", "kafrck-vccn013.domain.local:31093", "kafrck-vccn014.domain.local:31094"] + # message topic selection + partitioning + topic: '%{[fields.app_id]}' + #topic: 'metricbeat' + partition.round_robin: + reachable_only: false + required_acks: 1 + compression: gzip + max_message_bytes: 1000000 + # Optional fields that you can specify to add additional information to the + # output. + fields: + app_id: metricbeat_app_id +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + host: ${NODE_NAME} + labels.dedot: true + annotations.dedot: true + #hosts: ["localhost:10255"] + # If using Red Hat OpenShift remove the previous hosts entry and + # uncomment these settings: + hosts: ["https://${HOSTNAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt +--- +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:6.7.1 + imagePullPolicy: IfNotPresent + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 400Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-daemonset-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-daemonset-modules + - name: data + hostPath: + path: /var/lib/metricbeat-data + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-deployment-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + processors: + - add_cloud_metadata: + - add_kubernetes_metadata: + labels.dedot: true + annotations.dedot: true + - drop_fields: + when: + has_fields: ['kubernetes.labels.app'] + fields: + - 'kubernetes.labels.app' + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + output.kafka: + # initial brokers for reading cluster metadata + hosts: ["kafrck-vccn010.domain.local:31090", "kafrck-vccn011.domain.local:31091", "kafrck-vccn012.domain.local:31092", "kafrck-vccn013.domain.local:31093", "kafrck-vccn014.domain.local:31094"] + # message topic selection + partitioning + topic: '%{[fields.app_id]}' + #topic: 'metricbeat' + partition.round_robin: + reachable_only: false + required_acks: 1 + compression: gzip + max_message_bytes: 1000000 + + # Optional fields that you can specify to add additional information to the + # output. + fields: + app_id: metricbeat-kubernetes + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-deployment-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + # This module requires `kube-state-metrics` up and running under `kube-system` namespace + kubernetes.yml: |- + - module: kubernetes + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + # Uncomment this to get k8s events: + - event + period: 10s + host: ${NODE_NAME} + hosts: ["kube-state-metrics:8080"] +--- +# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:6.7.1 + imagePullPolicy: IfNotPresent + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + ] + env: + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 400Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: metricbeat-deployment-config + - name: modules + configMap: + defaultMode: 0600 + name: metricbeat-deployment-modules +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- \ No newline at end of file diff --git a/roles/helm-install/README.md b/roles/helm-install/README.md new file mode 100644 index 0000000..424d3ab --- /dev/null +++ b/roles/helm-install/README.md @@ -0,0 +1,28 @@ + +## Helm and Tiller installation + +### 2. Create helm tiller namespace and service-account: +``` +kubectl create ns kube-tiller +kubectl create serviceaccount tiller --namespace kube-tiller + +cat << EOF | kubectl create -f - +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tiller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: tiller + namespace: kube-tiller +EOF +``` + +### 3. Install Helm Chart package manager: +``` +helm init --service-account tiller --tiller-namespace kube-tiller +``` diff --git a/roles/ingress-nginx/tasks/main.yaml b/roles/ingress-nginx/tasks/main.yaml new file mode 100644 index 0000000..0bec19a --- /dev/null +++ b/roles/ingress-nginx/tasks/main.yaml @@ -0,0 +1,20 @@ +- name: copy ingress-nginx manifests + template: + src: ingress-nginx.yaml + dest: /tmp/ingress-nginx.yaml + force: yes + +- name: Set nginx-ingress-controller container image value + replace: + path: /tmp/ingress-nginx.yaml + regexp: "image: quay.io/kubernetes-ingress-controller" + replace: "image: {{ container_registry }}/system" + +- name: Set alpines container image value + replace: + path: /tmp/ingress-nginx.yaml + regexp: "image: privateregistry" + replace: "image: {{ container_registry }}/system" + +- name: Create nginx-ingress-controller Deployments + shell: kubectl apply -f /tmp/ingress-nginx.yaml diff --git a/roles/ingress-nginx/templates/ingress-nginx.yaml b/roles/ingress-nginx/templates/ingress-nginx.yaml new file mode 100644 index 0000000..6cdec37 --- /dev/null +++ b/roles/ingress-nginx/templates/ingress-nginx.yaml @@ -0,0 +1,280 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-ingress-nginx +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-configuration + namespace: kube-ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +data: + log-format-upstream: '"$http_x_forwarded_for" $remote_user [$time_local] "$request" $request_time $host $request_length $request_time $status "$http_referer" "$http_user_agent" "$the_real_ip" $cookie_WSPE_SID $cookie_CoreID6 $cookie_WSPFY $cookie_svi_dec $namespace' + worker-shutdown-timeout: "600s" + gzip-level: "1" + upstream-keepalive-connections: "128" + skip-access-log-urls: "/nginx_status" + large-client-header-buffers: "4 16k" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-ingress-serviceaccount + namespace: kube-ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nginx-ingress-clusterrole + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: nginx-ingress-role + namespace: kube-ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-nginx" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: nginx-ingress-role-nisa-binding + namespace: kube-ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-role +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount + namespace: kube-ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nginx-ingress-clusterrole-nisa-binding + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-ingress-clusterrole +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount + namespace: kube-ingress-nginx +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + namespace: kube-ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + annotations: + co.elastic.metrics/raw: '[{"metricsets":["collector"],"module":"prometheus","namespace":"kube-ingress-nginx","period":"10s","hosts":["${data.host}:10254"]}]' + spec: + serviceAccountName: nginx-ingress-serviceaccount + #nodeSelector: + # role: edge + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - ingress-nginx + topologyKey: kubernetes.io/hostname + initContainers: + - command: + - sh + - -c + - sysctl -w net.core.somaxconn=32768; sysctl -w net.ipv4.ip_local_port_range="32768 65535" + image: privateregistry/alpine:3.9 + imagePullPolicy: IfNotPresent + name: sysctl + securityContext: + privileged: true + containers: + - name: nginx-ingress-controller + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 + args: + - /nginx-ingress-controller + - --configmap=$(POD_NAMESPACE)/nginx-configuration + - --publish-service=$(POD_NAMESPACE)/ingress-nginx + - --enable-ssl-passthrough + - --annotations-prefix=nginx.ingress.kubernetes.io + - --v=1 + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + # www-data -> 33 + runAsUser: 33 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 +--- +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + namespace: kube-ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + type: NodePort + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + ports: + - name: https + nodePort: 30443 + port: 443 + targetPort: 443 + protocol: TCP +--- diff --git a/roles/kafka-charts/tasks/main.yaml b/roles/kafka-charts/tasks/main.yaml deleted file mode 100644 index 9eb8511..0000000 --- a/roles/kafka-charts/tasks/main.yaml +++ /dev/null @@ -1,31 +0,0 @@ -- name: Docker Pull Kafka Container Images - shell: docker pull confluentinc/cp-kafka:5.0.1 - delegate_to: "{{ item }}" - with_items: "{{ groups['k8s-workers'] }}" - -- name: Docker Pull Zookeeper Container Images - shell: docker pull confluentinc/cp-zookeeper:5.0.1 - delegate_to: "{{ item }}" - with_items: "{{ groups['k8s-workers'] }}" - -- name: Docker Pull Kafka-Prometheus Exporter Container Images - shell: docker pull solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 - delegate_to: "{{ item }}" - with_items: "{{ groups['k8s-workers'] }}" - -- name: Create kubernetes namespace - shell: kubectl create namespace congo-kafka-nonprod - ignore_errors: true - -- name: "Push Kafka charts to /tmp" - copy: - src: ./templates/ - dest: /tmp/charts/ - -- name: Helm install Zookeeper - shell: /usr/local/bin/helm install --name cp-zookeeper-nonprod --namespace congo-kafka-nonprod -f /tmp/charts/cp-zookeeper/values.yaml /tmp/charts/cp-zookeeper --tiller-namespace kube-tiller --namespace congo-kafka-nonprod - ignore_errors: true - -- name: Helm install Kafka Broker - shell: /usr/local/bin/helm install --name cp-kafka-nonprod --namespace congo-kafka-nonprod -f /tmp/charts/cp-kafka/values.yaml /tmp/charts/cp-kafka --tiller-namespace kube-tiller --namespace congo-kafka-nonprod - ignore_errors: true \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-kafka/.helmignore b/roles/kafka-charts/templates/cp-kafka/.helmignore deleted file mode 100755 index f0c1319..0000000 --- a/roles/kafka-charts/templates/cp-kafka/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/roles/kafka-charts/templates/cp-kafka/Chart.yaml b/roles/kafka-charts/templates/cp-kafka/Chart.yaml deleted file mode 100755 index 5615f92..0000000 --- a/roles/kafka-charts/templates/cp-kafka/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for Confluent Kafka on Kubernetes -name: cp-kafka -version: 0.1.0 diff --git a/roles/kafka-charts/templates/cp-kafka/README.md b/roles/kafka-charts/templates/cp-kafka/README.md deleted file mode 100755 index 29758c1..0000000 --- a/roles/kafka-charts/templates/cp-kafka/README.md +++ /dev/null @@ -1,207 +0,0 @@ -# CP-Kafka Helm Chart - -This chart bootstraps a cluster of Confluent Kafka - -## Prerequisites - -* Kubernetes 1.9.2+ -* Helm 2.8.2+ - -## Developing Environment: - -* [Pivotal Container Service (PKS)](https://pivotal.io/platform/pivotal-container-service) -* [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/) - -## Docker Image Source - -* [DockerHub -> ConfluentInc](https://hub.docker.com/u/confluentinc/) - -## Installing the Chart - -### Install along with cp-helm-charts - -```console -git clone https://github.com/confluentinc/cp-helm-charts.git -helm install cp-helm-charts -``` - -To install with a specific name, you can do: -```console -helm install --name my-confluent cp-helm-charts -``` - -### Install with a existing cp-zookeeper - -```console -helm install --set cp-zookeeper.enabled=false,cp-zookeeper.url="unhinged-robin-cp-zookeeper-headless:2181" cp-helm-charts/charts/cp-kafka -``` - -### Installed Components - -You can use `helm status ` to view all of the installed components. - -For example: - -```console -$ helm status garish-cat -NAMESPACE: default -STATUS: DEPLOYED - -RESOURCES: -==> v1/ConfigMap -NAME DATA AGE -boiling-heron-zookeeper-jmx-configmap 1 5m -boiling-heron-cp-kafka-jmx-configmap 1 5m - -==> v1/Service -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -boiling-heron-zookeeper-headless ClusterIP None 2888/TCP,3888/TCP 5m -boiling-heron-zookeeper ClusterIP 10.19.244.17 2181/TCP 5m -boiling-heron-0-external NodePort 10.19.240.13 19092:31090/TCP 5m -boiling-heron-1-external NodePort 10.19.243.241 19092:31091/TCP 5m -boiling-heron-2-external NodePort 10.19.248.189 19092:31092/TCP 5m -boiling-heron-cp-kafka-headless ClusterIP None 9092/TCP 5m -boiling-heron-cp-kafka ClusterIP 10.19.254.252 9092/TCP 5m - -==> v1beta1/StatefulSet -NAME DESIRED CURRENT AGE -boiling-heron-zookeeper 3 3 5m -boiling-heron-cp-kafka 3 3 5m - -==> v1beta1/PodDisruptionBudget -NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE -boiling-heron-zookeeper-pdb N/A 1 1 5m - -==> v1/Pod(related) -NAME READY STATUS RESTARTS AGE -boiling-heron-zookeeper-0 2/2 Running 0 5m -boiling-heron-zookeeper-1 2/2 Running 0 5m -boiling-heron-zookeeper-2 2/2 Running 0 5m -boiling-heron-cp-kafka-0 2/2 Running 0 5m -boiling-heron-cp-kafka-1 2/2 Running 0 5m -boiling-heron-cp-kafka-2 2/2 Running 0 5m -``` - -There are -1. A [Confluent Zookeeper Ensemble](https://github.com/confluentinc/cp-helm-charts/tree/master/charts/cp-zookeeper) created by cp-zookeeper chart. -1. A [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) `boiling-heron-cp-kafka` which contains 3 Kafka [Pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/): `boiling-heron-cp-kafka-<0|1|2>`. Each Pod has a container running a Kafka Broker and an optional sidecar JMX Exporter Container. -1. A [Service](https://kubernetes.io/docs/concepts/services-networking/service/) `boiling-heron-cp-kafka` for clients to connect to Kafka. -1. A [Headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) `boiling-heron-cp-kafka-headless` to control the network domain for the Kafka processes. -1. A group(N = number of brokers) of [NodePort Service](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport) `boiling-heron-cp-kafka-${i}-external` to allow access to Kafka Cluster from outside. -1. A [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) which contains configuration for Prometheus JMX Exporter. - -## Configuration - -You can specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. - -Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, - -```console -helm install --name my-kafka -f my-values.yaml ./cp-kafka -``` - -> **Tip**: A default [values.yaml](values.yaml) is provided - -### Kafka Cluster - -The configuration parameters in this section control the resources requested and utilized by the cp-kafka chart. - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `brokers` | The number of Broker servers. | `3` | - -### Image - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `image` | Docker Image of Confluent Kafka. | `confluentinc/cp-kafka` | -| `imageTag` | Docker Image Tag of Confluent Kafka. | `5.0.1` | -| `imagePullPolicy` | Docker Image Tag of Confluent Kafka. | `IfNotPresent` | -| `imagePullSecrets` | Secrets to be used for private registries. | see [values.yaml](values.yaml) for details | - -### StatefulSet Configurations - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `podManagementPolicy` | The Kafka StatefulSet Pod Management Policy: `Parallel` or `OrderedReady`. | `OrderedReady` | -| `updateStrategy` | The Kafka StatefulSet update strategy: `RollingUpdate` or `OnDelete`. | `RollingUpdate` | - -### Confluent Kafka Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `configurationOverrides` | Kafka [configuration](https://kafka.apache.org/documentation/#brokerconfigs) overrides in the dictionary format | `{}` | -| `customEnv` | Custom environmental variables | `{}` | - -### Persistence - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `persistence.enabled` | Whether to create a PVC. If `false`, an `emptyDir` on the host will be used. | `true` | -| `persistence.size` | Size for log dir, where Kafka will store log data. | `5Gi` | -| `persistence.storageClass` | Valid options: `nil`, `"-"`, or storage class name. | `nil` | -| `persistence.disksPerBroker` | The amount of disks that will be attached per instance of Kafka broker. | 1 | - -### Kafka JVM Heap Options - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `heapOptions` | The JVM Heap Options for Kafka | `"-Xms1G -Xmx1G"` | - -### Resources - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `resources.requests.cpu` | The amount of CPU to request. | see [values.yaml](values.yaml) for details | -| `resources.requests.memory` | The amount of memory to request. | see [values.yaml](values.yaml) for details | -| `resources.requests.limit` | The upper limit CPU usage for a Kafka Pod. | see [values.yaml](values.yaml) for details | -| `resources.requests.limit` | The upper limit memory usage for a Kafka Pod. | see [values.yaml](values.yaml) for details | - -### Annotations - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `podAnnotations` | Map of custom annotations to attach to the pod spec. | `{}` | - -### JMX Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `jmx.port` | The jmx port which JMX style metrics are exposed. | `5555` | - -### Prometheus JMX Exporter Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `prometheus.jmx.enabled` | Whether or not to install Prometheus JMX Exporter as a sidecar container and expose JMX metrics to Prometheus. | `true` | -| `prometheus.jmx.image` | Docker Image for Prometheus JMX Exporter container. | `solsson/kafka-prometheus-jmx-exporter@sha256` | -| `prometheus.jmx.imageTag` | Docker Image Tag for Prometheus JMX Exporter container. | `6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143` | -| `prometheus.jmx.port` | JMX Exporter Port which exposes metrics in Prometheus format for scraping. | `5556` | - -### External Access - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `nodeport.enabled` | Whether or not to allow access to kafka cluster from outside k8s through NodePort. | `false` | -| `nodeport.servicePort` | The Port broker will advertise to external producers and consumers. | `19092` | -| `nodeport.firstListenerPort` | The first NodePort that Kafka Broker will use for advertising to external producers and consumers. For each broker, advertise.listeners port for external will be set to `31090 + {index of broker pod}`. | `31090` | - -### Deployment Topology - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `nodeSelector` | Dictionary containing key-value-pairs to match labels on nodes. When defined pods will only be scheduled on nodes, that have each of the indicated key-value pairs as labels. Further information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) | `{}` -| `tolerations`| Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling. Further information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `{}` - -## Dependencies - -### Zookeeper - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `cp-zookeeper.enabled` | Whether or not to install cp-zookeeper chart alongside cp-kafka chart | `true` | -| `cp-zookeeper.persistence.enabled` | Whether to create a PVC. If `false`, an `emptyDir` on the host will be used. | `true` | -| `cp-zookeeper.persistence.dataDirSize` | Size for Data dir, where ZooKeeper will store the in-memory database snapshots. This will overwrite corresponding value in cp-zookeeper chart's value.yaml | `5Gi` | -| `cp-zookeeper.persistence.dataLogDirSize` | Size for data log dir, which is a dedicated log device to be used, and helps avoid competition between logging and snapshots. This will overwrite corresponding value in cp-zookeeper chart's value.yaml. | `5Gi` | -| `cp-zookeeper.url` | Service name of Zookeeper cluster (Not needed if zookeeper.enabled is set to true). | `""` | -| `cp-zookeeper.clientPort` | Port of Zookeeper Cluster | `2181` | diff --git a/roles/kafka-charts/templates/cp-kafka/templates/NOTES.txt b/roles/kafka-charts/templates/cp-kafka/templates/NOTES.txt deleted file mode 100755 index 3d72de1..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/NOTES.txt +++ /dev/null @@ -1,40 +0,0 @@ -This chart installs a {{ default 3 .Values.servers }} nodes Confluent Kafka Cluster. - -To connect from a client pod: - -1. Deploy a kafka client pod with configuration: - - apiVersion: v1 - kind: Pod - metadata: - name: kafka-client - namespace: {{ .Release.Namespace }} - spec: - containers: - - name: kafka-client - image: {{ .Values.image }}:{{ .Values.imageTag }} - command: - - sh - - -c - - "exec tail -f /dev/null" - -2. Log into the Pod - - kubectl exec -it kafka-client -- /bin/bash - -3. Explore with kafka commands: - - # Delete the topic if it exists - kafka-topics --zookeeper {{ template "cp-kafka.cp-zookeeper.service-name" . }} --topic {{ template "cp-kafka.fullname" . }}-topic --delete --if-exists - - # Create the topic - kafka-topics --zookeeper {{ template "cp-kafka.cp-zookeeper.service-name" . }} --topic {{ template "cp-kafka.fullname" . }}-topic --create --partitions 1 --replication-factor 1 --if-not-exists - - # Create a message - MESSAGE="`date -u`" - - # Produce a test message to the topic - echo "$MESSAGE" | kafka-console-producer --broker-list {{ template "cp-kafka.fullname" . }}:9092 --topic {{ template "cp-kafka.fullname" . }}-topic && \ - - # Consume a test message from the topic - kafka-console-consumer --bootstrap-server {{ template "cp-kafka.fullname" . }}-headless:9092 --topic {{ template "cp-kafka.fullname" . }}-topic --from-beginning --timeout-ms 2000 | grep "$MESSAGE" diff --git a/roles/kafka-charts/templates/cp-kafka/templates/_helpers.tpl b/roles/kafka-charts/templates/cp-kafka/templates/_helpers.tpl deleted file mode 100755 index f47e1bd..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/_helpers.tpl +++ /dev/null @@ -1,66 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "cp-kafka.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "cp-kafka.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "cp-kafka.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified zookeeper name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "cp-kafka.cp-zookeeper.fullname" -}} -{{- $name := default "cp-zookeeper" (index .Values "cp-zookeeper" "nameOverride") -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Form the Zookeeper URL. If zookeeper is installed as part of this chart, use k8s service discovery, -else use user-provided URL -*/}} -{{- define "cp-kafka.cp-zookeeper.service-name" }} -{{- if (index .Values "cp-zookeeper" "enabled") -}} -{{- printf "%s-headless:2181" (include "cp-kafka.cp-zookeeper.fullname" .) }} -{{- else -}} -{{- $zookeeperConnect := printf "%s" (index .Values "cp-zookeeper" "url") }} -{{- $zookeeperConnectOverride := (index .Values "configurationOverrides" "zookeeper.connect") }} -{{- default $zookeeperConnect $zookeeperConnectOverride }} -{{- end -}} -{{- end -}} - -{{/* -Create a variable containing all the datadirs created. -*/}} - -{{- define "cp-kafka.log.dirs" -}} -{{- range $k, $e := until (.Values.persistence.disksPerBroker|int) -}} -{{- if $k}}{{- printf ","}}{{end}} -{{- printf "/opt/kafka/data-%d/logs" $k -}} -{{- end -}} -{{- end -}} diff --git a/roles/kafka-charts/templates/cp-kafka/templates/headless-service.yaml b/roles/kafka-charts/templates/cp-kafka/templates/headless-service.yaml deleted file mode 100755 index 3538042..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/headless-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "cp-kafka.fullname" . }}-headless - labels: - app: {{ template "cp-kafka.name" . }} - chart: {{ template "cp-kafka.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - ports: - - port: 9092 - name: broker - clusterIP: None - selector: - app: {{ template "cp-kafka.name" . }} - release: {{ .Release.Name }} \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-kafka/templates/jmx-configmap.yaml b/roles/kafka-charts/templates/cp-kafka/templates/jmx-configmap.yaml deleted file mode 100755 index fde14cb..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/jmx-configmap.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if and .Values.prometheus.jmx.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "cp-kafka.fullname" . }}-jmx-configmap - labels: - app: {{ template "cp-kafka.name" . }} - chart: {{ template "cp-kafka.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - jmx-kafka-prometheus.yml: |+ - jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:{{ .Values.jmx.port }}/jmxrmi - lowercaseOutputName: true - lowercaseOutputLabelNames: true - ssl: false - rules: - - pattern : kafka.server<>(Value|OneMinuteRate) - name: "cp_kafka_server_replicamanager_$1" - - pattern : kafka.controller<>Value - name: "cp_kafka_controller_kafkacontroller_$1" - - pattern : kafka.server<>OneMinuteRate - name: "cp_kafka_server_brokertopicmetrics_$1" - - pattern : kafka.network<>OneMinuteRate - name: "cp_kafka_network_requestmetrics_requestspersec_$1" - - pattern : kafka.network<>Value - name: "cp_kafka_network_socketserver_networkprocessoravgidlepercent" - - pattern : kafka.server<>Value - name: "cp_kafka_server_replicafetchermanager_maxlag_$1" - - pattern : kafka.server<>OneMinuteRate - name: "cp_kafka_kafkarequesthandlerpool_requesthandleravgidlepercent" - - pattern : kafka.controller<>OneMinuteRate - name: "cp_kafka_controller_controllerstats_$1" - - pattern : kafka.server<>OneMinuteRate - name: "cp_kafka_server_sessionexpirelistener_$1" -{{- end }} \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-kafka/templates/nodeport-service.yaml b/roles/kafka-charts/templates/cp-kafka/templates/nodeport-service.yaml deleted file mode 100755 index 963fd85..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/nodeport-service.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{- if .Values.nodeport.enabled }} - {{- $fullName := include "cp-kafka.fullname" . }} - {{- $brokers := .Values.brokers | int }} - {{- $servicePort := .Values.nodeport.servicePort }} - {{- $root := . }} - {{- range $i, $e := until $brokers }} - {{- $externalListenerPort := add $root.Values.nodeport.firstListenerPort $i }} - {{- $responsiblePod := printf "%s-%d" (printf "%s" $fullName) $i }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $root.Release.Name }}-{{ $i }}-nodeport - labels: - app: {{ include "cp-kafka.name" $root }} - chart: {{ template "cp-kafka.chart" $root }} - release: {{ $root.Release.Name }} - heritage: {{ $root.Release.Service }} - pod: {{ $responsiblePod }} -spec: - type: NodePort - ports: - - name: external-broker - port: {{ $servicePort }} - targetPort: {{ $externalListenerPort }} - nodePort: {{ $externalListenerPort }} - protocol: TCP - selector: - app: {{ include "cp-kafka.name" $root }} - release: {{ $root.Release.Name }} - statefulset.kubernetes.io/pod-name: {{ $responsiblePod | quote }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-kafka/templates/service.yaml b/roles/kafka-charts/templates/cp-kafka/templates/service.yaml deleted file mode 100755 index 169d0cc..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "cp-kafka.fullname" . }} - labels: - app: {{ template "cp-kafka.name" . }} - chart: {{ template "cp-kafka.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - ports: - - port: 9092 - name: broker - selector: - app: {{ template "cp-kafka.name" . }} - release: {{ .Release.Name }} \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-kafka/templates/statefulset.yaml b/roles/kafka-charts/templates/cp-kafka/templates/statefulset.yaml deleted file mode 100755 index 259dad5..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/statefulset.yaml +++ /dev/null @@ -1,177 +0,0 @@ -{{- $advertisedListenersOverride := first (pluck "advertised.listeners" .Values.configurationOverrides) }} -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: {{ template "cp-kafka.fullname" . }} - labels: - app: {{ template "cp-kafka.name" . }} - chart: {{ template "cp-kafka.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - serviceName: {{ template "cp-kafka.fullname" . }}-headless - podManagementPolicy: {{ .Values.podManagementPolicy }} - replicas: {{ default 3 .Values.brokers }} - updateStrategy: - type: {{ .Values.updateStrategy }} - template: - metadata: - labels: - app: {{ template "cp-kafka.name" . }} - release: {{ .Release.Name }} - {{- if or .Values.podAnnotations .Values.prometheus.jmx.enabled }} - annotations: - {{- range $key, $value := .Values.podAnnotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- if .Values.prometheus.jmx.enabled }} - prometheus.io/scrape: "true" - prometheus.io/port: {{ .Values.prometheus.jmx.port | quote }} - {{- end }} - {{- end }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - {{ template "cp-kafka.name" . }} - - key: "release" - operator: In - values: - - {{ .Release.Name }} - topologyKey: "kubernetes.io/hostname" - containers: - {{- if .Values.prometheus.jmx.enabled }} - - name: prometheus-jmx-exporter - image: "{{ .Values.prometheus.jmx.image }}:{{ .Values.prometheus.jmx.imageTag }}" - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - {{ .Values.prometheus.jmx.port | quote }} - - /etc/jmx-kafka/jmx-kafka-prometheus.yml - ports: - - containerPort: {{ .Values.prometheus.jmx.port }} - resources: -{{ toYaml .Values.prometheus.jmx.resources | indent 10 }} - volumeMounts: - - name: jmx-config - mountPath: /etc/jmx-kafka - {{- end }} - - name: {{ template "cp-kafka.name" . }}-broker - image: "{{ .Values.image }}:{{ .Values.imageTag }}" - imagePullPolicy: "{{ .Values.imagePullPolicy }}" - ports: - - containerPort: 9092 - name: kafka - {{- if .Values.prometheus.jmx.enabled }} - - containerPort: {{ .Values.jmx.port }} - name: jmx - {{- end }} - {{- if .Values.nodeport.enabled }} - {{- $brokers := .Values.brokers | int }} - {{- $root := . }} - {{- range $i, $e := until $brokers }} - - containerPort: {{ add $root.Values.nodeport.firstListenerPort $i }} - name: nodeport-{{ $i }} - {{- end }} - {{- end }} - resources: -{{ toYaml .Values.resources | indent 10 }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: KAFKA_HEAP_OPTS - value: {{ .Values.heapOptions }} - - name: KAFKA_ZOOKEEPER_CONNECT - value: {{ include "cp-kafka.cp-zookeeper.service-name" . | quote }} - - name: KAFKA_LOG_DIRS - value: {{ include "cp-kafka.log.dirs" . | quote }} - {{- range $key, $value := .Values.configurationOverrides }} - - name: {{ printf "KAFKA_%s" $key | replace "." "_" | upper | quote }} - value: {{ $value | quote }} - {{- end }} - {{- range $key, $value := .Values.customEnv }} - - name: {{ $key | quote }} - value: {{ $value | quote }} - {{- end }} - {{- if .Values.jmx.port }} - - name: KAFKA_JMX_PORT - value: "{{ .Values.jmx.port }}" - {{- end }} - # This is required because the Downward API does not yet support identification of - # pod numbering in statefulsets. Thus, we are required to specify a command which - # allows us to extract the pod ID for usage as the Kafka Broker ID. - # See: https://github.com/kubernetes/kubernetes/issues/31218 - command: - - sh - - -exc - - | - export KAFKA_BROKER_ID=${HOSTNAME##*-} && \ - export KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${POD_IP}:9092{{ if kindIs "string" $advertisedListenersOverride }}{{ printf ",%s" $advertisedListenersOverride }}{{ end }} && \ - exec /etc/confluent/docker/run - volumeMounts: - {{- $disksPerBroker := .Values.persistence.disksPerBroker | int }} - {{- range $k, $e := until $disksPerBroker }} - - name: datadir-{{$k}} - mountPath: /opt/kafka/data-{{$k}} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: -{{ toYaml .Values.imagePullSecrets | indent 8 }} - {{- end }} - volumes: - {{- if not .Values.persistence.enabled }} - - name: datadir - emptyDir: {} - {{- end }} - {{- if .Values.prometheus.jmx.enabled }} - - name: jmx-config - configMap: - name: {{ template "cp-kafka.fullname" . }}-jmx-configmap - {{- end }} - {{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} - {{- end }} - {{- if .Values.tolerations }} - tolerations: -{{ toYaml .Values.tolerations | indent 8 }} - {{- end }} - {{- if .Values.persistence.enabled }} - volumeClaimTemplates: - {{- $disksPerBroker := .Values.persistence.disksPerBroker | int }} - {{- $root := . }} - {{- range $k, $e := until $disksPerBroker }} - - metadata: - name: datadir-{{$k}} - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: "{{ $root.Values.persistence.size }}" - {{- if $root.Values.persistence.storageClass }} - {{- if (eq "-" $root.Values.persistence.storageClass) }} - storageClassName: "" - {{- else }} - storageClassName: "{{ $root.Values.persistence.storageClass }}" - {{- end }} - {{- end }} -{{- end }} -{{- end }} diff --git a/roles/kafka-charts/templates/cp-kafka/templates/tests/canary-pod.yaml b/roles/kafka-charts/templates/cp-kafka/templates/tests/canary-pod.yaml deleted file mode 100755 index ce871d7..0000000 --- a/roles/kafka-charts/templates/cp-kafka/templates/tests/canary-pod.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ .Release.Name }}-canary" - annotations: - "helm.sh/hook": test-success - "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" -spec: - containers: - - name: {{ .Release.Name }}-canary - image: "{{ .Values.image }}:{{ .Values.imageTag }}" - imagePullPolicy: "{{ .Values.imagePullPolicy }}" - command: - - sh - - -c - - | - # Delete the topic if it exists - kafka-topics --zookeeper {{ template "cp-kafka.cp-zookeeper.service-name" . }} --topic {{ template "cp-kafka.fullname" . }}-canary-topic --delete --if-exists - # Create the topic - kafka-topics --zookeeper {{ template "cp-kafka.cp-zookeeper.service-name" . }} --topic {{ template "cp-kafka.fullname" . }}-canary-topic --create --partitions 1 --replication-factor 1 --if-not-exists && \ - # Create a message - MESSAGE="`date -u`" && \ - # Produce a test message to the topic - echo "$MESSAGE" | kafka-console-producer --broker-list {{ template "cp-kafka.fullname" . }}:9092 --topic {{ template "cp-kafka.fullname" . }}-canary-topic && \ - # Consume a test message from the topic - kafka-console-consumer --bootstrap-server {{ template "cp-kafka.fullname" . }}-headless:9092 --topic {{ template "cp-kafka.fullname" . }}-canary-topic --from-beginning --timeout-ms 2000 | grep "$MESSAGE" - restartPolicy: Never \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-kafka/values.yaml b/roles/kafka-charts/templates/cp-kafka/values.yaml deleted file mode 100755 index d27cf3e..0000000 --- a/roles/kafka-charts/templates/cp-kafka/values.yaml +++ /dev/null @@ -1,137 +0,0 @@ -# Default values for cp-kafka. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -## ------------------------------------------------------ -## Kafka -## ------------------------------------------------------ - -## Number of Kafka brokers -brokers: 3 - -## Image Info -## ref: https://hub.docker.com/r/confluentinc/cp-kafka/ -image: confluentinc/cp-kafka -imageTag: 5.0.1 - -## Specify a imagePullPolicy -## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images -imagePullPolicy: IfNotPresent - -## Specify an array of imagePullSecrets. -## Secrets must be manually created in the namespace. -## ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod -imagePullSecrets: - -## StatefulSet Config -## Start and stop pods in Parallel or OrderedReady (one-by-one.) -## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy -podManagementPolicy: OrderedReady - -## The StatefulSet Update Strategy which Kafka will use when changes are applied: OnDelete or RollingUpdate -## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies -updateStrategy: RollingUpdate - -## Kafka Server properties -## ref: https://kafka.apache.org/documentation/#configuration -configurationOverrides: - "offsets.topic.replication.factor": "3" - # "default.replication.factor": 3 - # "min.insync.replicas": 2 - # "auto.create.topics.enable": false - - ## Options required for external access via NodePort - ## ref: - ## - http://kafka.apache.org/documentation/#security_configbroker - ## - https://cwiki.apache.org/confluence/display/KAFKA/KIP-103%3A+Separation+of+Internal+and+External+traffic - ## - ## Setting "advertised.listeners" here appends to "PLAINTEXT://${POD_IP}:9092," - "advertised.listeners": |- - EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) - "listener.security.protocol.map": |- - PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - -## Additional env variables -customEnv: {} - -persistence: - enabled: true - - ## The size of the PersistentVolume to allocate to each Kafka Pod in the StatefulSet. For - ## production servers this number should likely be much larger. - size: 1536Gi - - ## Kafka data Persistent Volume Storage Class - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - storageClass: "local-storage-kafkabroker" - - disksPerBroker: 3 - -## Kafka JVM Heap Option -heapOptions: "-Xms28G -Xmx28G" - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -## Custom pod annotations -podAnnotations: {} - -## Node labels for pod assignment -## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ -nodeSelector: {} - -## Taints to tolerate on node assignment: -## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -tolerations: {} - -## Monitoring -## Kafka JMX Settings -## ref: https://docs.confluent.io/current/kafka/monitoring.html -jmx: - port: 5555 - -## Prometheus Exporter Configuration -## ref: https://prometheus.io/docs/instrumenting/exporters/ -prometheus: - ## JMX Exporter Configuration - ## ref: https://github.com/prometheus/jmx_exporter - jmx: - enabled: true - image: solsson/kafka-prometheus-jmx-exporter@sha256 - imageTag: 6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 - port: 5556 - -nodeport: - enabled: true - servicePort: 19092 - firstListenerPort: 31090 - -## ------------------------------------------------------ -## Zookeeper -## ------------------------------------------------------ -cp-zookeeper: - ## If true, install the cp-zookeeper chart alongside cp-kafka - ## ref: ../cp-zookeeper - enabled: false - servers: 3 - persistence: - enabled: true - dataDirSize: 5Gi - dataLogDirSize: 5Gi - - ## If the Zookeeper Chart is disabled a URL and port are required to connect - url: "cp-zookeeper-nonprod:2181" diff --git a/roles/kafka-charts/templates/cp-zookeeper/.helmignore b/roles/kafka-charts/templates/cp-zookeeper/.helmignore deleted file mode 100755 index f0c1319..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/roles/kafka-charts/templates/cp-zookeeper/Chart.yaml b/roles/kafka-charts/templates/cp-zookeeper/Chart.yaml deleted file mode 100755 index 1da3393..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for Confluent Zookeeper on Kubernetes -name: cp-zookeeper -version: 0.1.0 \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-zookeeper/README.md b/roles/kafka-charts/templates/cp-zookeeper/README.md deleted file mode 100755 index 2666bef..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/README.md +++ /dev/null @@ -1,193 +0,0 @@ -# CP-Zookeeper Helm Chart - -This chart bootstraps an ensemble of Confluent Zookeeper - -## Prerequisites - -* Kubernetes 1.9.2+ -* Helm 2.8.2+ - -## Developing Environment - -* [Pivotal Container Service (PKS)](https://pivotal.io/platform/pivotal-container-service) -* [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/) - -## Docker Image Source - -* [DockerHub -> ConfluentInc](https://hub.docker.com/u/confluentinc/) - -## Installing the Chart - -### Install along with cp-helm-charts - -```console -git clone https://github.com/confluentinc/cp-helm-charts.git -helm install cp-helm-charts -``` - -To install with a specific name, you can do: - -```console -helm install --name my-zookeeper ./cp-zookeeper -``` - -### Install with cp-zookeeper alone - -```console -helm install cp-helm-charts/charts/cp-zookeeper -``` - -### Installed Components - -You can use `helm status ` to view all of the installed components. - -For example: - -```console{%raw} -$ helm status unsung-salamander - -RESOURCES: -==> v1/Service -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -unsung-salamander-cp-zookeeper-headless ClusterIP None 2888/TCP,3888/TCP 6m -unsung-salamander-cp-zookeeper ClusterIP 10.19.241.242 2181/TCP 6m - -==> v1beta1/StatefulSet -NAME DESIRED CURRENT AGE -unsung-salamander-cp-zookeeper 3 3 6m - -==> v1beta1/PodDisruptionBudget -NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE -unsung-salamander-cp-zookeeper-pdb N/A 1 1 6m - -==> v1/Pod(related) -NAME READY STATUS RESTARTS AGE -unsung-salamander-cp-zookeeper-0 1/1 Running 0 6m -unsung-salamander-cp-zookeeper-1 1/1 Running 0 6m -unsung-salamander-cp-zookeeper-2 1/1 Running 0 6m - -==> v1/ConfigMap -NAME DATA AGE -unsung-salamander-cp-zookeeper-jmx-configmap 1 6m -``` - -There are -1. A [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) `unsung-salamander-cp-zookeeper` which contains 3 Zookeeper [Pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/): `unsung-salamander-cp-zookeeper-<0|1|2>`. Each Pod has a container running a ZooKeeper server and an optional sidecar JMX Exporter Container. -1. A [PodDisruptionBudget](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) `unsung-salamander-cp-zookeeper-pdb` to ensure service availability during planned maintenance. -1. A [Service](https://kubernetes.io/docs/concepts/services-networking/service/) `unsung-salamander-cp-zookeeper` for clients to connect to Zookeeper. -1. A [Headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) `unsung-salamander-cp-zookeeper-headless` to control the network domain for the ZooKeeper processes. -1. A [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) which contains configuration for Prometheus JMX Exporter. - -## Configuration - -You can specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. - -Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, - -```console -helm install --name my-zookeeper -f my-values.yaml ./cp-zookeeper -``` - -> **Tip**: A default [values.yaml](values.yaml) is provided - -### Zookeeper Ensemble - -The configuration parameters in this section control the resources requested and utilized by the cp-zookeeper chart. - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `servers` | The number of ZooKeeper servers. This should always be (1,3,5, or 7). | `3` | - -### PodDisruptionBudget - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `minAvailable` | The minimum number of servers that must be available during evictions. This should in the interval `[(servers/2) + 1,(servers - 1)]`. If not set, `maxUnavailable: 1` will be applied. | `servers-1` | - -### Image - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `image` | Docker Image of Confluent Zookeeper. | `confluentinc/cp-zookeeper` | -| `imageTag` | Docker Image Tag of Confluent Zookeeper. | `5.0.1` | -| `imagePullPolicy` | Docker Image Tag of Confluent Zookeeper. | `IfNotPresent` | -| `imagePullSecrets` | Secrets to be used for private registries. | see [values.yaml](values.yaml) for details | - -### StatefulSet Configurations - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `podManagementPolicy` | The Zookeeper StatefulSet Pod Management Policy: `Parallel` or `OrderedReady`. | `OrderedReady` | -| `updateStrategy` | The ZooKeeper StatefulSet update strategy: `RollingUpdate` or `OnDelete`. | `RollingUpdate` | - -### Confluent Zookeeper Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `tickTime` | The length of a single tick, which is the basic time unit used by ZooKeeper, as measured in milliseconds. It is used to regulate heartbeats, and timeouts. For example, the minimum session timeout will be two ticks. | `2000` | -| `syncLimit` | Amount of time, in ticks (see `tickTime`), to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped. | `5` | -| `initLimit` | Amount of time, in ticks (see `tickTime`), to allow followers to connect and sync to a leader. Increased this value as needed, if the amount of data managed by ZooKeeper is large. | `10` | -| `maxClientCnxns` | Limits the number of concurrent connections (at the socket level) that a single client, identified by IP address, may make to a single member of the ZooKeeper ensemble. This is used to prevent certain classes of DoS attacks, including file descriptor exhaustion. | `60` | -| `autoPurgeSnapRetainCount` | When enabled, ZooKeeper auto purge feature retains the autopurge.snapRetainCount most recent snapshots and the corresponding transaction logs in the dataDir and dataLogDir respectively and deletes the rest. | `3` | -| `autoPurgePurgeInterval` | The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging. | `72` | - -### Zookeeper JVM Heap Options - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `heapOptions` | The JVM Heap Options for Zookeeper | `"-Xms512M -Xmx512M"` | - -### Port - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `serverPort` | The port on which the ZooKeeper servers listen for requests from other servers in the ensemble. | `2888` | -| `leaderElectionPort` | The port on which the ZooKeeper servers perform leader election. | `3888` | -| `clientPort` | The port to listen for client connections; that is, the port that clients attempt to connect to. | `2181` | - -### Persistence - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `persistence.enabled` | Whether to create a PVC. If `false`, an `emptyDir` on the host will be used. | `true` | -| `persistence.dataDirSize` | Size for Data dir, where ZooKeeper will store the in-memory database snapshots. | `5Gi` | -| `persistence.dataDirStorageClass` | Valid options: `nil`, `"-"`, or storage class name. | `nil` | -| `persistence.dataLogDirSize` | Size for data log dir, which is a dedicated log device to be used, and helps avoid competition between logging and snaphots. | `5Gi` | -| `persistence.dataLogDirStorageClass` | Valid options: `nil`, `"-"`, or storage class name. | `nil` | - -### Resources - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `resources.requests.cpu` | The amount of CPU to request. | see [values.yaml](values.yaml) for details | -| `resources.requests.memory` | The amount of memory to request. | see [values.yaml](values.yaml) for details | -| `resources.requests.limit` | The upper limit CPU usage for a Zookeeper Pod. | see [values.yaml](values.yaml) for details | -| `resources.requests.limit` | The upper limit memory usage for a Zookeeper Pod. | see [values.yaml](values.yaml) for details | - -### Annotations - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `podAnnotations` | Map of custom annotations to attach to the pod spec. | `{}` | - -### JMX Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `jmx.port` | The jmx port which JMX style metrics are exposed. | `5555` | - -### Prometheus JMX Exporter Configuration - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `prometheus.jmx.enabled` | Whether or not to install Prometheus JMX Exporter as a sidecar container and expose JMX metrics to Prometheus. | `true` | -| `prometheus.jmx.image` | Docker Image for Prometheus JMX Exporter container. | `solsson/kafka-prometheus-jmx-exporter@sha256` | -| `prometheus.jmx.imageTag` | Docker Image Tag for Prometheus JMX Exporter container. | `6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143` | -| `prometheus.jmx.port` | JMX Exporter Port which exposes metrics in Prometheus format for scraping. | `5556` | - -### Deployment Topology - -| Parameter | Description | Default | -| --------- | ----------- | ------- | -| `nodeSelector` | Dictionary containing key-value-pairs to match labels on nodes. When defined pods will only be scheduled on nodes, that have each of the indicated key-value pairs as labels. Further information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) | `{}` -| `tolerations`| Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling. Further information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `{}` diff --git a/roles/kafka-charts/templates/cp-zookeeper/templates/NOTES.txt b/roles/kafka-charts/templates/cp-zookeeper/templates/NOTES.txt deleted file mode 100755 index d6025b9..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/templates/NOTES.txt +++ /dev/null @@ -1,34 +0,0 @@ -This chart installs a {{ default 3 .Values.servers }} nodes Confluent Zookeeper ensemble for Confluent Kafka Cluster. - -Connection string for Confluent Kafka: - {{ template "cp-zookeeper.fullname" . }}-0.{{ template "cp-zookeeper.fullname" . }}-headless:{{ .Values.clientPort }},{{ template "cp-zookeeper.fullname" . }}-1.{{ template "cp-zookeeper.fullname" . }}-headless:{{ .Values.clientPort }},... - -To connect from a client pod: - -1. Deploy a zookeeper client pod with configuration: - - apiVersion: v1 - kind: Pod - metadata: - name: zookeeper-client - namespace: {{ .Release.Namespace }} - spec: - containers: - - name: zookeeper-client - image: {{ .Values.image }}:{{ .Values.imageTag }} - command: - - sh - - -c - - "exec tail -f /dev/null" - -2. Log into the Pod - - kubectl exec -it zookeeper-client -- /bin/bash - -3. Use zookeeper-shell to connect in the zookeeper-client Pod: - - zookeeper-shell {{ template "cp-zookeeper.fullname" . }}:{{ .Values.clientPort }} - -4. Explore with zookeeper commands: - - https://zookeeper.apache.org/doc/current/zookeeperStarted.html#sc_ConnectingToZooKeeper diff --git a/roles/kafka-charts/templates/cp-zookeeper/templates/_helpers.tpl b/roles/kafka-charts/templates/cp-zookeeper/templates/_helpers.tpl deleted file mode 100755 index b54ba03..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/templates/_helpers.tpl +++ /dev/null @@ -1,48 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "cp-zookeeper.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "cp-zookeeper.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "cp-zookeeper.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a server list string based on fullname, namespace, # of servers -in a format like "zkhost1:port:port;zkhost2:port:port" -*/}} -{{- define "cp-zookeeper.serverlist" -}} -{{- $namespace := .Release.Namespace }} -{{- $name := include "cp-zookeeper.fullname" . -}} -{{- $serverPort := .Values.serverPort -}} -{{- $leaderElectionPort := .Values.leaderElectionPort -}} -{{- $zk := dict "servers" (list) -}} -{{- range $idx, $v := until (int .Values.servers) }} -{{- $noop := printf "%s-%d.%s-headless.%s:%d:%d" $name $idx $name $namespace (int $serverPort) (int $leaderElectionPort) | append $zk.servers | set $zk "servers" -}} -{{- end }} -{{- printf "%s" (join ";" $zk.servers) | quote -}} -{{- end -}} diff --git a/roles/kafka-charts/templates/cp-zookeeper/templates/headless-service.yaml b/roles/kafka-charts/templates/cp-zookeeper/templates/headless-service.yaml deleted file mode 100755 index cd95dfb..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/templates/headless-service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "cp-zookeeper.fullname" . }}-headless - labels: - app: {{ template "cp-zookeeper.name" . }} - chart: {{ template "cp-zookeeper.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - ports: - - port: {{ .Values.serverPort }} - name: server - - port: {{ .Values.leaderElectionPort }} - name: leader-election - clusterIP: None - selector: - app: {{ template "cp-zookeeper.name" . }} - release: {{ .Release.Name }} \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-zookeeper/templates/jmx-configmap.yaml b/roles/kafka-charts/templates/cp-zookeeper/templates/jmx-configmap.yaml deleted file mode 100755 index 1732bf4..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/templates/jmx-configmap.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if and .Values.prometheus.jmx.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "cp-zookeeper.fullname" . }}-jmx-configmap - labels: - app: {{ template "cp-zookeeper.name" . }} - chart: {{ template "cp-zookeeper.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - jmx-zookeeper-prometheus.yml: |+ - jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:{{ .Values.jmx.port }}/jmxrmi - lowercaseOutputName: true - lowercaseOutputLabelNames: true - ssl: false - rules: - - pattern: "org.apache.ZooKeeperService<>(\\w+)" - name: "cp_zookeeper_$2" - - pattern: "org.apache.ZooKeeperService<>(\\w+)" - name: "cp_zookeeper_$3" - labels: - replicaId: "$2" - - pattern: "org.apache.ZooKeeperService<>(\\w+)" - name: "cp_zookeeper_$4" - labels: - replicaId: "$2" - memberType: "$3" - - pattern: "org.apache.ZooKeeperService<>(\\w+)" - name: "cp_zookeeper_$4_$5" - labels: - replicaId: "$2" - memberType: "$3" -{{- end }} \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-zookeeper/templates/poddisruptionbudget.yaml b/roles/kafka-charts/templates/cp-zookeeper/templates/poddisruptionbudget.yaml deleted file mode 100755 index bb5e1ad..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/templates/poddisruptionbudget.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: {{ template "cp-zookeeper.fullname" . }}-pdb - labels: - app: {{ template "cp-zookeeper.name" . }} - chart: {{ template "cp-zookeeper.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - matchLabels: - app: {{ template "cp-zookeeper.name" . }} - release: {{ .Release.Name }} - {{- if .Values.minAvailable }} - minAvailable: {{ .Values.minAvailable }} - {{- else }} - maxUnavailable: 1 - {{- end }} diff --git a/roles/kafka-charts/templates/cp-zookeeper/templates/service.yaml b/roles/kafka-charts/templates/cp-zookeeper/templates/service.yaml deleted file mode 100755 index 8d97a20..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "cp-zookeeper.fullname" . }} - labels: - app: {{ template "cp-zookeeper.name" . }} - chart: {{ template "cp-zookeeper.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - type: {{ .Values.serviceType }} - ports: - - port: {{ .Values.clientPort }} - name: client - selector: - app: {{ template "cp-zookeeper.name" . }} - release: {{ .Release.Name }} \ No newline at end of file diff --git a/roles/kafka-charts/templates/cp-zookeeper/templates/statefulset.yaml b/roles/kafka-charts/templates/cp-zookeeper/templates/statefulset.yaml deleted file mode 100755 index 20a2dd9..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/templates/statefulset.yaml +++ /dev/null @@ -1,178 +0,0 @@ -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: {{ template "cp-zookeeper.fullname" . }} - labels: - app: {{ template "cp-zookeeper.name" . }} - chart: {{ template "cp-zookeeper.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - serviceName: {{ template "cp-zookeeper.fullname" . }}-headless - podManagementPolicy: {{ .Values.podManagementPolicy }} - replicas: {{ default 3 .Values.servers }} - updateStrategy: - type: {{ .Values.updateStrategy }} - template: - metadata: - labels: - app: {{ template "cp-zookeeper.name" . }} - release: {{ .Release.Name }} - {{- if or .Values.podAnnotations .Values.prometheus.jmx.enabled }} - annotations: - {{- range $key, $value := .Values.podAnnotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- if .Values.prometheus.jmx.enabled }} - prometheus.io/scrape: "true" - prometheus.io/port: {{ .Values.prometheus.jmx.port | quote }} - {{- end }} - {{- end }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - {{ template "cp-zookeeper.name" . }} - - key: "release" - operator: In - values: - - {{ .Release.Name }} - topologyKey: "kubernetes.io/hostname" - containers: - {{- if .Values.prometheus.jmx.enabled }} - - name: prometheus-jmx-exporter - image: "{{ .Values.prometheus.jmx.image }}:{{ .Values.prometheus.jmx.imageTag }}" - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - {{ .Values.prometheus.jmx.port | quote }} - - /etc/jmx-zookeeper/jmx-zookeeper-prometheus.yml - ports: - - containerPort: {{ .Values.prometheus.jmx.port }} - resources: -{{ toYaml .Values.prometheus.jmx.resources | indent 10 }} - volumeMounts: - - name: jmx-config - mountPath: /etc/jmx-zookeeper - {{- end }} - - name: {{ template "cp-zookeeper.name" . }}-server - image: "{{ .Values.image }}:{{ .Values.imageTag }}" - imagePullPolicy: "{{ .Values.imagePullPolicy }}" - ports: - - containerPort: {{ .Values.clientPort }} - name: client - - containerPort: {{ .Values.serverPort }} - name: server - - containerPort: {{ .Values.leaderElectionPort }} - name: leader-election - {{- if .Values.prometheus.jmx.enabled }} - - containerPort: {{ .Values.jmx.port }} - name: jmx - {{- end }} - resources: -{{ toYaml .Values.resources | indent 10 }} - env: - - name : KAFKA_HEAP_OPTS - value: "{{ .Values.heapOptions }}" - {{- if .Values.jmx.port }} - - name : KAFKA_JMX_PORT - value: "{{ .Values.jmx.port }}" - {{- end }} - - name : ZOOKEEPER_TICK_TIME - value: "{{ .Values.tickTime }}" - - name : ZOOKEEPER_SYNC_LIMIT - value: "{{ .Values.syncLimit }}" - - name : ZOOKEEPER_INIT_LIMIT - value: "{{ .Values.initLimit }}" - - name : ZOOKEEPER_MAX_CLIENT_CNXNS - value: "{{ .Values.maxClientCnxns }}" - - name : ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT - value: "{{ .Values.autoPurgeSnapRetainCount}}" - - name : ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL - value: "{{ .Values.autoPurgePurgeInterval}}" - - name: ZOOKEEPER_CLIENT_PORT - value: "{{ .Values.clientPort }}" - - name : ZOOKEEPER_SERVERS - value: {{ template "cp-zookeeper.serverlist" . }} - # ZOOKEEPER_SERVER_ID is required just to pass cp-zookeeper ensure script for env check, - # the value(metadata.mame) is not used and will be overwritten in command part - - name: ZOOKEEPER_SERVER_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - command: - - "bash" - - "-c" - - "ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-}+1)) && /etc/confluent/docker/run" - volumeMounts: - - name: datadir - mountPath: /var/lib/zookeeper/data - - name: datalogdir - mountPath: /var/lib/zookeeper/log - {{- if .Values.imagePullSecrets }} - imagePullSecrets: -{{ toYaml .Values.imagePullSecrets | indent 8 }} - {{- end }} - volumes: - {{ if not .Values.persistence.enabled }} - - name: datadir - emptyDir: {} - - name: datalogdir - emptyDir: {} - {{- end }} - {{- if .Values.prometheus.jmx.enabled }} - - name: jmx-config - configMap: - name: {{ template "cp-zookeeper.fullname" . }}-jmx-configmap - {{- end }} - {{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} - {{- end }} - {{- if .Values.tolerations }} - tolerations: -{{ toYaml .Values.tolerations | indent 8 }} - {{- end }} - {{- if .Values.persistence.enabled }} - volumeClaimTemplates: - - metadata: - name: datadir - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: "{{ .Values.persistence.dataDirSize }}" - {{- if .Values.persistence.dataDirStorageClass }} - {{- if (eq "-" .Values.persistence.dataDirStorageClass) }} - storageClassName: "" - {{- else }} - storageClassName: "{{ .Values.persistence.dataDirStorageClass }}" - {{- end }} - {{- end }} - - metadata: - name: datalogdir - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: "{{ .Values.persistence.dataLogDirSize }}" - {{- if .Values.persistence.dataLogDirStorageClass }} - {{- if (eq "-" .Values.persistence.dataLogDirStorageClass) }} - storageClassName: "" - {{- else }} - storageClassName: "{{ .Values.persistence.dataLogDirStorageClass }}" - {{- end }} - {{- end }} - {{- end }} diff --git a/roles/kafka-charts/templates/cp-zookeeper/values.yaml b/roles/kafka-charts/templates/cp-zookeeper/values.yaml deleted file mode 100755 index 57363b5..0000000 --- a/roles/kafka-charts/templates/cp-zookeeper/values.yaml +++ /dev/null @@ -1,114 +0,0 @@ -## Default values for cp-zookeeper. -## This is a YAML-formatted file. -## Declare variables to be passed into your templates. - -## ------------------------------------------------------ -## Zookeeper -## ------------------------------------------------------ - -## Number of zookeeper servers, should be odd number -servers: 3 - -## Minimum number of servers available to ensure the availability of zookeeper service -## minAvailable: 2 - -## Images Info -## ref: https://hub.docker.com/r/confluentinc/cp-zookeeper/ -image: confluentinc/cp-zookeeper -imageTag: 5.0.1 - -## Specify a imagePullPolicy -## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images -imagePullPolicy: IfNotPresent - -## Specify an array of imagePullSecrets. -## Secrets must be manually created in the namespace. -## ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod -imagePullSecrets: - -## StatefulSet Config -## Start and stop pods in Parallel or OrderedReady (one-by-one.) -## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy -podManagementPolicy: OrderedReady - -## The StatefulSet Update Strategy which Zookeeper will use when changes are applied: OnDelete or RollingUpdate -## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies -updateStrategy: RollingUpdate - -## Zookeeper Configuration -## ref: https://zookeeper.apache.org/doc/r3.4.10/zookeeperAdmin.html#sc_configuration -## ref: https://docs.confluent.io/current/zookeeper/deployment.html#important-configuration-options -tickTime: 2000 -syncLimit: 5 -initLimit: 10 -maxClientCnxns: 60 -autoPurgeSnapRetainCount: 3 -autoPurgePurgeInterval: 24 - -## Zookeeper JVM Heap Option -heapOptions: "-Xms2048M -Xmx2048M" - -## Port -## ref: https://zookeeper.apache.org/doc/r3.4.10/zookeeperAdmin.html#sc_configuration -serverPort: 2888 -leaderElectionPort: 3888 -clientPort: 2181 - -## Persistent Volumes -## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ -persistence: - enabled: true - - ## The size of the PersistentVolume to allocate to each Zookeeper Pod in the StatefulSet. For - ## production servers this number should likely be much larger. - ## - ## Size for Data dir, where ZooKeeper will store the in-memory database snapshots. - dataDirSize: 100Gi - dataDirStorageClass: "local-storage-zookeeperdata" - - ## Size for data log dir, which is a dedicated log device to be used, and helps avoid competition between logging and snaphots. - dataLogDirSize: 20Gi - dataLogDirStorageClass: "local-storage-zookeeperlog" - - -## Pod Compute Resources -## ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -## Custom pod annotations -podAnnotations: {} - -## Node labels for pod assignment -## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ -nodeSelector: {} - -## Taints to tolerate on node assignment: -## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -tolerations: {} - -## Monitoring -## Zookeeper JMX Settings -## ref: https://docs.confluent.io/current/installation/docker/docs/operations/monitoring.html -jmx: - port: 5555 - -## Prometheus Exporter Configuration -## ref: https://prometheus.io/docs/instrumenting/exporters/ -prometheus: - ## JMX Exporter Configuration - ## ref: https://github.com/prometheus/jmx_exporter - jmx: - enabled: true - image: solsson/kafka-prometheus-jmx-exporter@sha256 - imageTag: 6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 - port: 5556 diff --git a/roles/kubeadm-init-master/tasks/main.yaml b/roles/kubeadm-init-master/tasks/main.yaml index 7319093..0bb505d 100644 --- a/roles/kubeadm-init-master/tasks/main.yaml +++ b/roles/kubeadm-init-master/tasks/main.yaml @@ -1,6 +1,6 @@ -- name: Download Kubeadm container images - shell: kubeadm config images pull +- pause: + prompt: "Wait before initializing master" - name: "Create kubeadm init config yaml" copy: @@ -11,6 +11,10 @@ - name: Kubeadm init shell: kubeadm init --config=/tmp/kubeadm-config.yaml register: rslt + ignore_errors: yes + +- pause: + prompt: "Validate kubeadm init" - name: Store init output action: copy content="{{ rslt.stdout }}" dest="/etc/kubernetes/kubeadm-init.stdout" @@ -69,4 +73,4 @@ dest: /tmp/kubeadm-ha/{{item}} flat: yes with_items: - - admin.conf \ No newline at end of file + - admin.conf diff --git a/roles/kubeadm-join-masters/tasks/main.yaml b/roles/kubeadm-join-masters/tasks/main.yaml index 0d822f1..384b3ed 100755 --- a/roles/kubeadm-join-masters/tasks/main.yaml +++ b/roles/kubeadm-join-masters/tasks/main.yaml @@ -1,6 +1,3 @@ -- name: Download Kubeadm container images - shell: kubeadm config images pull - - name: "Create kubernetes folders" file: path: /etc/kubernetes/pki/etcd @@ -27,7 +24,7 @@ kubeadm_join: "{{ kubeadm_join_cmd.stdout }}" - name: Join Master replicas to cluster - shell: "{{ kubeadm_join }} --experimental-control-plane" + shell: "{{ kubeadm_join }} --control-plane" - pause: prompt: "Wait for containers to Pull and install" \ No newline at end of file diff --git a/roles/kubeadm-prep/tasks/main.yaml b/roles/kubeadm-prep/tasks/main.yaml index c0d1643..e033ea2 100755 --- a/roles/kubeadm-prep/tasks/main.yaml +++ b/roles/kubeadm-prep/tasks/main.yaml @@ -11,20 +11,35 @@ command: setenforce 0 ignore_errors: True -- name: Install kubeadm packages +- name: Install kubelet package become: yes yum: - name: "{{item}}" - allow_downgrade: yes - with_items: - - kubelet-{{kubelet_version}} - - kubeadm - - kubectl - -- name: Add netbridge config ip6 + name: "kubelet-{{kubeadm_version}}" + enablerepo: kubernetes + +- name: Install kubectl package + become: yes + yum: + name: "kubectl-{{kubeadm_version}}" + enablerepo: kubernetes + +- name: Install kubeadm package + become: yes + yum: + name: "kubeadm-{{kubeadm_version}}" + enablerepo: kubernetes + +- name: Add vm swappiness lineinfile: path: /etc/sysctl.d/k8s.conf - line: 'net.bridge.bridge-nf-call-ip6tables = 1' + line: 'vm.swappiness = 0' + state: present + create: yes + +- name: Add vm overcommit_memory + lineinfile: + path: /etc/sysctl.d/k8s.conf + line: 'vm.overcommit_memory = 1' state: present create: yes @@ -35,6 +50,13 @@ state: present create: yes +- name: Increase net ipv4 tcp_max_syn_backlog + lineinfile: + path: /etc/sysctl.d/k8s.conf + line: 'net.ipv4.tcp_max_syn_backlog=2621440' + state: present + create: yes + - name: update sysctl command: sysctl --system diff --git a/roles/kubeadm-singlenode-cluster/tasks/main.yaml b/roles/kubeadm-singlenode-cluster/tasks/main.yaml new file mode 100644 index 0000000..65d7b1c --- /dev/null +++ b/roles/kubeadm-singlenode-cluster/tasks/main.yaml @@ -0,0 +1,4 @@ +- name: kubectl taint remove master + shell: kubectl taint node {{ hostvars[item].inventory_hostname_short }} node-role.kubernetes.io/master- + with_items: "{{ groups['k8s-master-primary'][0] }}" + ignore_errors: True \ No newline at end of file diff --git a/roles/kubeadm-upgrade-masters/tasks/main.yaml b/roles/kubeadm-upgrade-masters/tasks/main.yaml new file mode 100755 index 0000000..7fe4f67 --- /dev/null +++ b/roles/kubeadm-upgrade-masters/tasks/main.yaml @@ -0,0 +1,50 @@ +- name: Upgrade kubenetes packages + yum: + name: "{{ packages }}" + enablerepo: kubernetes + vars: + packages: + - kubelet-{{kubeadm_version}} + - kubectl-{{kubeadm_version}} + - kubeadm-{{kubeadm_version}} + +- name: "Update kubeadm config yaml" + copy: + content: "{{ kubeadm_config_options | to_nice_yaml }}" + dest: /tmp/kubeadm-config.yaml + mode: 0644 + +- name: "Kubeadm upgrade plan" + shell: kubeadm upgrade plan --config /tmp/kubeadm-config.yaml --print-config + register: rslt + +- pause: + prompt: "Validate plan" + +- name: "Kubeadm upgrade dry-run" + shell: kubeadm upgrade apply {{kubernetes_version}} --config /tmp/kubeadm-config.yaml --dry-run + register: rslt + +- pause: + prompt: "Validate dry-run" + +- name: "Kubeadm upgrade" + shell: kubeadm upgrade apply -f --config /tmp/kubeadm-config.yaml {{kubernetes_version}} + register: rslt + +- pause: + prompt: "Proceed with restart" + +- name: Restart kubelet + systemd: + state: restarted + daemon_reload: yes + name: kubelet + enabled: yes + +- name: Upgrade masters sequentially + include_tasks: "upgrade_masters.yaml" + with_items: "{{ groups['k8s-master-replicas'] }}" + when: "hostvars[host_item].inventory_hostname == inventory_hostname" + loop_control: + loop_var: host_item \ No newline at end of file diff --git a/roles/kubeadm-upgrade-masters/tasks/upgrade_masters.yaml b/roles/kubeadm-upgrade-masters/tasks/upgrade_masters.yaml new file mode 100755 index 0000000..a0f5ed1 --- /dev/null +++ b/roles/kubeadm-upgrade-masters/tasks/upgrade_masters.yaml @@ -0,0 +1,23 @@ +- name: Upgrade kubenetes packages + yum: + name: "{{ packages }}" + enablerepo: kubernetes + vars: + packages: + - kubelet-{{kubeadm_version}} + - kubectl-{{kubeadm_version}} + - kubeadm-{{kubeadm_version}} + +- name: "Kubeadm upgrade: kubeadm upgrade node experimental-control-plane - remove (experimental-control-plane) for +v1.15" + shell: kubeadm upgrade node experimental-control-plane + register: rslt + +- pause: + prompt: "Proceed with restart" + +- name: Restart kubelet + systemd: + state: restarted + daemon_reload: yes + name: kubelet + enabled: yes \ No newline at end of file diff --git a/roles/kubeadm-upgrade-workers/tasks/main.yaml b/roles/kubeadm-upgrade-workers/tasks/main.yaml new file mode 100755 index 0000000..57f1690 --- /dev/null +++ b/roles/kubeadm-upgrade-workers/tasks/main.yaml @@ -0,0 +1,7 @@ + +- name: Upgrade workers sequentially + include_tasks: "upgrade_worker.yaml" + with_items: "{{ groups['k8s-workers'] }}" + when: "hostvars[host_item].inventory_hostname == inventory_hostname" + loop_control: + loop_var: host_item \ No newline at end of file diff --git a/roles/kubernetes-dashboard/README.md b/roles/kubernetes-dashboard/README.md new file mode 100644 index 0000000..8df7574 --- /dev/null +++ b/roles/kubernetes-dashboard/README.md @@ -0,0 +1,3 @@ + +curl https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta3/aio/deploy/recommended.yaml -O + diff --git a/roles/kubernetes-dashboard/tasks/main.yaml b/roles/kubernetes-dashboard/tasks/main.yaml index c9f786c..bcc1784 100644 --- a/roles/kubernetes-dashboard/tasks/main.yaml +++ b/roles/kubernetes-dashboard/tasks/main.yaml @@ -1,12 +1,14 @@ -- name: Docker Pull Container Images - shell: docker pull {{ kubernetes_dashboard_image }} - delegate_to: "{{ item }}" - with_items: "{{ groups['k8s-workers'] }}" +- name: Copy kubernetes-dashboard.yaml manifests + template: + src: kubernetes-dashboard.yaml + dest: /tmp/kubernetes-dashboard.yaml + force: yes -- name: Create Kubernetes Dashboard Deployment - shell: curl https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml | kubectl apply -f - - environment: "{{proxy_env}}" - -- pause: - prompt: "Wait for containers to Pull and install" +- name: Set kubernetes-dashboard container image value + replace: + path: /tmp/kubernetes-dashboard.yaml + regexp: "image: kubernetesui" + replace: "image: {{ container_registry }}/keystone" +- name: Create kubernetes-dashboard Deployment + shell: kubectl apply -f /tmp/kubernetes-dashboard.yaml diff --git a/roles/kubernetes-dashboard/templates/recommended.yaml b/roles/kubernetes-dashboard/templates/recommended.yaml new file mode 100644 index 0000000..f4f8448 --- /dev/null +++ b/roles/kubernetes-dashboard/templates/recommended.yaml @@ -0,0 +1,287 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: kubernetes-dashboard + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + ports: + - port: 443 + targetPort: 8443 + selector: + k8s-app: kubernetes-dashboard + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: kubernetes-dashboard +type: Opaque + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-csrf + namespace: kubernetes-dashboard +type: Opaque +data: + csrf: "" + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-key-holder + namespace: kubernetes-dashboard +type: Opaque + +--- + +kind: ConfigMap +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-settings + namespace: kubernetes-dashboard + +--- + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +rules: + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics. + - apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster", "dashboard-metrics-scraper"] + verbs: ["proxy"] + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] + verbs: ["get"] + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard +rules: + # Allow Metrics Scraper to get metrics from the Metrics server + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard + namespace: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + containers: + - name: kubernetes-dashboard + image: kubernetesui/dashboard:v2.0.0-beta3 + imagePullPolicy: Always + ports: + - containerPort: 8443 + protocol: TCP + args: + - --auto-generate-certificates + - --namespace=kubernetes-dashboard + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + +--- + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + ports: + - port: 8000 + targetPort: 8000 + selector: + k8s-app: dashboard-metrics-scraper + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: dashboard-metrics-scraper + template: + metadata: + labels: + k8s-app: dashboard-metrics-scraper + spec: + containers: + - name: dashboard-metrics-scraper + image: kubernetesui/metrics-scraper:v1.0.1 + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + scheme: HTTP + path: / + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + serviceAccountName: kubernetes-dashboard + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + volumes: + - name: tmp-volume + emptyDir: {} diff --git a/roles/local-storage-provisioner/tasks/main.yaml b/roles/local-storage-provisioner/tasks/main.yaml index 308fe0a..290afd7 100644 --- a/roles/local-storage-provisioner/tasks/main.yaml +++ b/roles/local-storage-provisioner/tasks/main.yaml @@ -1,13 +1,9 @@ -- name: Docker Pull Container Images - shell: docker pull {{ local_volume_provisioner_image }} - delegate_to: "{{ item }}" - with_items: "{{ groups['k8s-workers'] }}" - - name: copy local-storage-provisioner manifests template: src: local-storage-provisioner.yaml dest: /tmp/local-storage-provisioner.yaml + force: yes - name: Create Local Storage Provisioner Deployments shell: kubectl apply -f /tmp/local-storage-provisioner.yaml \ No newline at end of file diff --git a/roles/local-storage-provisioner/templates/local-storage-provisioner.yaml b/roles/local-storage-provisioner/templates/local-storage-provisioner.yaml index 6f14873..4775088 100644 --- a/roles/local-storage-provisioner/templates/local-storage-provisioner.yaml +++ b/roles/local-storage-provisioner/templates/local-storage-provisioner.yaml @@ -3,44 +3,31 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: local-storage-kafkabroker -provisioner: kubernetes.io/no-provisioner -volumeBindingMode: WaitForFirstConsumer ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local-storage-zookeeperdata -provisioner: kubernetes.io/no-provisioner -volumeBindingMode: WaitForFirstConsumer ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local-storage-zookeeperlog + name: local-storage-elastic provisioner: kubernetes.io/no-provisioner volumeBindingMode: WaitForFirstConsumer +# Supported policies: Delete, Retain +reclaimPolicy: Delete --- # Source: provisioner/templates/provisioner.yaml + apiVersion: v1 kind: ConfigMap metadata: name: local-provisioner-config namespace: default data: - useNodeNameOnly: "true" storageClassMap: | - local-storage-kafkabroker: - hostDir: /mnt/local-storage-kafkabroker - mountDir: /mnt/local-storage-kafkabroker - local-storage-zookeeperdata: - hostDir: /mnt/local-storage-zookeeperdata - mountDir: /mnt/local-storage-zookeeperdata - local-storage-zookeeperlog: - hostDir: /mnt/local-storage-zookeeperlog - mountDir: /mnt/local-storage-zookeeperlog + local-storage-elastic: + hostDir: /pv + mountDir: /pv + blockCleanerCommand: + - "/scripts/shred.sh" + - "2" + volumeMode: Filesystem + fsType: ext4 --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: local-volume-provisioner @@ -58,7 +45,7 @@ spec: spec: serviceAccountName: local-storage-admin containers: - - image: "quay.io/external_storage/local-volume-provisioner:v2.2.0" + - image: "container-registry01.nonprod.domain.local/keystone/local-volume-provisioner:v2.3.2" imagePullPolicy: "IfNotPresent" name: provisioner securityContext: @@ -71,32 +58,29 @@ spec: volumeMounts: - mountPath: /etc/provisioner/config name: provisioner-config - readOnly: true - - mountPath: /mnt/local-storage-kafkabroker - name: local-storage-kafkabroker - - mountPath: /mnt/local-storage-zookeeperdata - name: local-storage-zookeeperdata - - mountPath: /mnt/local-storage-zookeeperlog - name: local-storage-zookeeperlog + readOnly: true + - mountPath: /pv + name: local-storage-elastic + mountPropagation: "HostToContainer" volumes: - name: provisioner-config configMap: name: local-provisioner-config - - name: local-storage-kafkabroker - hostPath: - path: /mnt/local-storage-kafkabroker - - name: local-storage-zookeeperdata + - name: local-storage-elastic hostPath: - path: /mnt/local-storage-zookeeperdata - - name: local-storage-zookeeperlog - hostPath: - path: /mnt/local-storage-zookeeperlog + path: /pv --- +# Source: provisioner/templates/provisioner-service-account.yaml + apiVersion: v1 kind: ServiceAccount metadata: name: local-storage-admin + namespace: default + --- +# Source: provisioner/templates/provisioner-cluster-role-binding.yaml + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -134,19 +118,3 @@ roleRef: kind: ClusterRole name: local-storage-provisioner-node-clusterrole apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: Service -metadata: - name: local-volume-provisioner - namespace: default - labels: - app: local-volume-provisioner -spec: - type: ClusterIP - selector: - app: local-volume-provisioner - ports: - - name: metrics - port: 8080 - protocol: TCP \ No newline at end of file diff --git a/roles/os-patch-updates/tasks/main.yml b/roles/os-patch-updates/tasks/main.yml new file mode 100644 index 0000000..a181e50 --- /dev/null +++ b/roles/os-patch-updates/tasks/main.yml @@ -0,0 +1,9 @@ + +- name: OS patch hosts sequentially + include_tasks: "os-patches.yml" + with_items: + - "{{ groups['k8s-masters'] }}" + - "{{ groups['k8s-workers'] }}" + when: "hostvars[host_item].inventory_hostname == inventory_hostname" + loop_control: + loop_var: host_item diff --git a/roles/os-patch-updates/tasks/os-patches.yml b/roles/os-patch-updates/tasks/os-patches.yml new file mode 100644 index 0000000..de26521 --- /dev/null +++ b/roles/os-patch-updates/tasks/os-patches.yml @@ -0,0 +1,32 @@ +- pause: + prompt: "Set kubectl context to cluster and proceed with draining node {{ inventory_hostname_short }}" + +- name: Drain kubernetes worker node {{ inventory_hostname_short }} + local_action: command kubectl drain {{ inventory_hostname_short }} --delete-local-data --ignore-daemonsets --force + register: kubeadm_drain + until: kubeadm_drain.rc == 0 + become: no + +- name: Update the system + yum: + name: "*" + state: latest + disable_gpg_check: true + +- name: Check for reboot + shell: if [ $(rpm -q kernel|tail -n 1) != kernel-$(uname -r) ]; then echo 'reboot' else echo 'no'; fi + ignore_errors: true + register: reboot_hint + changed_when: "'reboot' in reboot_hint.stdout" + +- name: Reboot system to complete patching + reboot: + reboot_timeout: 300 + ignore_errors: true + when: reboot_hint.stdout.find("reboot") != -1 + register: rebooting + +- name: Uncordon worker node + local_action: command kubectl uncordon {{ inventory_hostname_short }} + register: kubeadm_uncordon + become: no diff --git a/roles/repos/tasks/main.yaml b/roles/repos/tasks/main.yaml index 401ff1a..c8810f7 100755 --- a/roles/repos/tasks/main.yaml +++ b/roles/repos/tasks/main.yaml @@ -13,16 +13,16 @@ yum_repository: name: docker-ce description: docker-ce stable repo - baseurl: https://download.docker.com/linux/centos/7/$basearch/stable - gpgkey: https://download.docker.com/linux/centos/gpg - gpgcheck: yes - proxy: "http://127.0.0.1:8888" + baseurl: https://pkgs.domain.local/linux/docker-ce/ + gpgcheck: no + sslverify: no + enabled: false - name: Add kubernetes repo yum_repository: name: kubernetes description: Kubernetes repo - baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch - gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg - gpgcheck: yes - proxy: "http://127.0.0.1:8888" + baseurl: https://pkgs.domain.local/linux/kubernetes/yum/repos/kubernetes-el7-$basearch + gpgcheck: no + sslverify: no + enabled: false diff --git a/roles/uninstall/tasks/main.yaml b/roles/uninstall/tasks/main.yaml index 92d3057..50db0f6 100755 --- a/roles/uninstall/tasks/main.yaml +++ b/roles/uninstall/tasks/main.yaml @@ -10,8 +10,6 @@ ip link delete cni0 rm -rf /var/lib/docker/* rm -rf /etc/kubernetes - rm -f /tmp/kubeadm-config.yaml - rm -rf /tmp/charts/ - name: remove yum packages become: yes @@ -24,7 +22,7 @@ - kubectl - docker-ce - device-mapper-persistent-data - - lvm2 +# - lvm2 - name: Remove docker-ce-stable repo yum_repository: