initial commit

.gitignore

@@ -0,0 +1,5 @@
+.DS_Store
+*.csr
+*.key
+*secret.yaml
+key.pem

README.md

@@ -0,0 +1,52 @@
+## Ansible Playbooks to install an air gapped Highly Available Kubernetes cluster
+Uses Kubeadm with the --experimental-control-plane switch to install the Stacked Control Plane. This deployment pattern is well suited to single purpose, secure, infrastructure software clusters with low frequency container changes.
+
+- Uses recently GA'd Kubeadm 1.13 HA multi-master joining features
+- Uses the Ansible Host as a short lived forward proxy to download cluster software and container images
+- No container registry is required (downloaded images are cache to the local docker engines)
+
+
+### Prerequisites:
+
+* Setup ssh access from Ansible host to Kubernetes nodes.
+```ssh-copy-id -i ~/.ssh/id_rsa.pub <user@host>```
+- Setup a local forward proxy with Internet access on the Ansible host (e.g. tinyproxy). Update the proxy environment variable details in `inventory/group_vars/all`.
+
+![Air Gapped Network Flows](./airgapped-ansible-kubernetes.png)
+
+
+### Environment preparation:
+
+Specify the Master and Worker hostnames in the `inventory/cluster` file:
+
+Update the `inventory/group_vars` sections:
+- choose the desired versions for kubernetes and docker
+- setup the pod network cidr (default setup is for calico - modify in calico.yaml as well)
+- specify the version of Helm to use
+- specify the Local Storage Provisioner version
+
+
+### Install process:
+
+Run install-all.yaml playbook to get your cluster fully setup. Note 8888 below if the default tinyproxy port- adjust according to your proxy install.
+You can also run the different playbooks separately for different purposes (setting up docker, masters, kubeadm, helm ...).
+
+```
+ansible-playbook -i inventory/cluster playbooks/install-all.yaml --private-key=~/.ssh/id_rsa -u username --extra-vars ansible_ssh_extra_args="-R8888:localhost:8888" -v
+```
+
+### What install-all.yaml includes:
+
+- Adds the required yum repositories
+- Installs docker
+- Installs kubeadm, kubelet and kubectl
+- Initializes the first master with etcd and kubernetes-api
+- Installs Calico networking
+- Joins replica master nodes to the primary master
+- Adds the worker nodes to the cluster
+- Installs Helm & Tiller
+- Install Kubernetes Dashboard (Metrics Server support coming 02/19)
+- Installs Local Storage Provisioner (usefull for cloud native, shared nothing, stateful set apps such as Kafka, Zookeeper, Elastic)
+
+### Still to do:
+- Update Replica Master /etc/kubernetes/* file permissions after Ansible copy

ansible.cfg

@@ -0,0 +1,7 @@
+[defaults]
+host_key_checking = False
+forks = 12
+inventory = inventory
+roles_path = roles
+display_skipped_hosts = false
+any_errors_fatal = true

inventory/cluster

@@ -0,0 +1,19 @@
+[k8s-master-primary]
+master1
+
+[k8s-master-replicas]
+master2
+master3
+
+[k8s-masters:children]
+k8s-master-primary
+k8s-master-replicas
+
+[k8s-workers]
+worker1
+worker2
+worker3
+
+[k8s-nodes:children]
+k8s-masters
+k8s-workers

inventory/group_vars/all

@@ -0,0 +1,41 @@
+---
+
+docker_version: 18.09.1
+kubernetes_version: v1.13.1
+kubelet_version: 1.13.1
+
+# proxy server to tunnel through SSH to ansible deployment host
+proxy_env:
+    http_proxy: http://localhost:8888
+    https_proxy: http://localhost:8888
+    no_proxy: 127.0.0.1,10.0.0.0
+
+# kubernetes API load balanced VIP for HA installations
+kubernetes_loadbalanced_api_dns: k8sapi01.domain.local
+
+# Docker Daemon configuration
+docker_ce_daemon_options:
+    exec-opts: [ "native.cgroupdriver=systemd" ]
+    log-driver: json-file
+    log-opts:
+        max-size: "100m"
+        max-file: "7"
+    storage-driver: overlay2
+    storage-opts: [ "overlay2.override_kernel_check=true" ]
+
+# Kubernetes Kubeadm Cluster Configuration
+kubeadm_config_options:
+    apiVersion: kubeadm.k8s.io/v1beta1
+    kind: ClusterConfiguration
+    kubernetesVersion: "{{ kubernetes_version }}"
+    apiServer:
+        certSANs:
+            - "{{ kubernetes_loadbalanced_api_dns }}"
+    controlPlaneEndpoint: "{{ kubernetes_loadbalanced_api_dns }}:6443"
+    networking:
+        podSubnet: 10.244.0.0/16
+
+# Addon Container Images
+tiller_image: gcr.io/kubernetes-helm/tiller:v2.12.1
+local_volume_provisioner_image: quay.io/external_storage/local-volume-provisioner:v2.2.0
+kubernetes_dashboard_image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1

playbooks/ansible-requirements.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-nodes
+  become: yes
+  roles:
+    - ansible-requirements

playbooks/calico.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-master-primary
+  become: yes
+  roles:
+    - calico

playbooks/docker.yaml

@@ -0,0 +1,6 @@
+- hosts: k8s-nodes
+  become: yes
+  roles:
+    - docker
+
+

playbooks/helm-install.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-master-primary
+  become: yes
+  roles:
+    - helm-install

playbooks/install-all.yaml

@@ -0,0 +1,11 @@
+- import_playbook: ansible-requirements.yaml
+- import_playbook: repos.yaml
+- import_playbook: docker.yaml
+- import_playbook: kubeadm-prep.yaml
+- import_playbook: kubeadm-init-master.yaml
+- import_playbook: calico.yaml
+- import_playbook: kubeadm-join-masters.yaml
+- import_playbook: kubeadm-join-workers.yaml
+- import_playbook: helm-install.yaml
+- import_playbook: local-storage-provisioner.yaml
+- import_playbook: kubernetes-dashboard.yaml

playbooks/kafka-charts.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-master-primary
+  become: yes
+  roles:
+    - kafka-charts

playbooks/kubeadm-init-master.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-master-primary
+  become: yes
+  roles:
+    - kubeadm-init-master

playbooks/kubeadm-join-masters.yaml

@@ -0,0 +1,5 @@
+- hosts: k8s-master-replicas
+  become: yes
+  roles:
+    - kubeadm-join-masters
+    #- {role: kubeadm-join-master, when: "groups['k8s-masters'][0] != inventory_hostname"}

playbooks/kubeadm-join-workers.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-workers
+  become: yes
+  roles:
+    - kubeadm-join-workers

playbooks/kubeadm-prep.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-nodes
+  become: yes
+  roles:
+    - kubeadm-prep

playbooks/kubernetes-dashboard.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-master-primary
+  become: yes
+  roles:
+    - kubernetes-dashboard

playbooks/local-storage-provisioner.yaml

@@ -0,0 +1,4 @@
+- hosts: k8s-master-primary
+  become: yes
+  roles:
+    - local-storage-provisioner

playbooks/repos.yaml

@@ -0,0 +1,6 @@
+- hosts: k8s-nodes
+  become: yes
+  roles:
+    - repos
+
+

playbooks/uninstall.yaml

@@ -0,0 +1,6 @@
+- hosts: k8s-nodes
+  become: yes
+  roles:
+    - uninstall
+
+

roles/ansible-requirements/tasks/main.yml

@@ -0,0 +1,16 @@
+- name: Check if yum is installed.
+  raw: yum --version
+  register: has_yum
+  failed_when: False
+
+- name: Clean yum
+  raw: yum clean all
+  when: has_yum.rc == 0
+
+- name: Install python2 [yum].
+  raw: yum install -y python
+  register: installed_via_yum
+  when: has_yum.rc == 0
+
+- name: Ensure python2 has been installed properly.
+  raw: test -e /usr/bin/python

roles/calico/tasks/main.yaml

@@ -0,0 +1,18 @@
+- name: Copy Calico RBAC rbac-kdd.yaml manifests
+  template:
+    src: rbac-kdd.yaml
+    dest: /tmp/rbac-kdd.yaml
+
+- name: Create Calico RBAC Deployment
+  shell: kubectl apply -f /tmp/rbac-kdd.yaml
+
+- name: Copy calico.yaml manifests
+  template:
+    src: calico.yaml
+    dest: /tmp/calico.yaml
+
+- name: Create Calico Deployment
+  shell: kubectl apply -f /tmp/calico.yaml
+
+- pause:
+    prompt: "Wait for containers to Pull and install"