forked from Altinity/clickhouse-operator
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathVagrantfile
212 lines (173 loc) · 8.8 KB
/
Vagrantfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
# -*- mode: ruby -*-
# vi: set ft=ruby :
def total_cpus
require 'etc'
Etc.nprocessors
end
Vagrant.configure(2) do |config|
config.vm.box = "generic/ubuntu2004"
config.vm.box_check_update = false
config.vm.synced_folder ".", "/vagrant"
if Vagrant.has_plugin?("vagrant-vbguest")
config.vbguest.auto_update = false
end
if Vagrant.has_plugin?("vagrant-timezone")
config.timezone.value = "UTC"
end
config.vm.provider "virtualbox" do |vb|
vb.gui = false
vb.cpus = total_cpus
vb.memory = "6144"
vb.default_nic_type = "virtio"
vb.customize ["modifyvm", :id, "--uartmode1", "file", File::NULL ]
vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
vb.customize ["modifyvm", :id, "--ioapic", "on"]
vb.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000]
end
config.vm.define :clickhouse_operator do |clickhouse_operator|
clickhouse_operator.vm.network "private_network", ip: "172.16.2.99", nic_type: "virtio"
# port forwarding works only when pair with kubectl port-forward
# grafana
clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 3000, host_ip: "127.0.0.1", host: 3000
# mertics-exporter
clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 8888, host_ip: "127.0.0.1", host: 8888
# prometheus
clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 9090, host_ip: "127.0.0.1", host: 9090
# alertmanager
clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 9093, host_ip: "127.0.0.1", host: 9093
clickhouse_operator.vm.host_name = "local-altinity-clickhouse-operator"
# vagrant plugin install vagrant-disksize
clickhouse_operator.disksize.size = '50GB'
end
config.vm.provision "shell", inline: <<-SHELL
set -xeuo pipefail
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install --no-install-recommends -y apt-transport-https ca-certificates software-properties-common curl
apt-get install --no-install-recommends -y htop ethtool mc curl wget jq socat git ntp
# yq
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
add-apt-repository ppa:rmescandon/yq
apt-get install --no-install-recommends -y yq
# clickhouse
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E0C56BD4
add-apt-repository "deb http://repo.clickhouse.tech/deb/stable/ main/"
apt-get install --no-install-recommends -y clickhouse-client
# docker
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8D81803C0EBFCD88
add-apt-repository "deb https://download.docker.com/linux/ubuntu focal test"
apt-get install --no-install-recommends -y docker-ce pigz
# docker compose
apt-get install -y --no-install-recommends python3-distutils
curl -sL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
python3 /tmp/get-pip.py
pip3 install -U setuptools
pip3 install -U docker-compose
# k9s CLI
K9S_VERSION=$(curl -sL https://github.com/derailed/k9s/releases/latest -H "Accept: application/json" | jq -r .tag_name)
wget -c --progress=bar:force:noscroll -O /usr/local/bin/k9s_${K9S_VERSION}_Linux_x86_64.tar.gz https://github.com/derailed/k9s/releases/download/${K9S_VERSION}/k9s_Linux_x86_64.tar.gz
curl -sL https://github.com/derailed/k9s/releases/download/${K9S_VERSION}/checksums.txt | grep Linux_x86_64.tar.gz > /usr/local/bin/k9s.sha256
sed -i -e "s/k9s_Linux_x86_64\.tar\.gz/\\/usr\\/local\\/bin\\/k9s_${K9S_VERSION}_Linux_x86_64\\.tar\\.gz/g" /usr/local/bin/k9s.sha256
sha256sum -c /usr/local/bin/k9s.sha256
tar --verbose -zxvf /usr/local/bin/k9s_${K9S_VERSION}_Linux_x86_64.tar.gz -C /usr/local/bin k9s
# minikube
MINIKUBE_VERSION=1.17.1
wget -c --progress=bar:force:noscroll -O /usr/local/bin/minikube https://github.com/kubernetes/minikube/releases/download/v${MINIKUBE_VERSION}/minikube-linux-amd64
chmod +x /usr/local/bin/minikube
# required for k8s 1.18+
apt-get install -y conntrack
# K8S_VERSION=${K8S_VERSION:-1.14.10}
# export VALIDATE_YAML=false # only for 1.14
# K8S_VERSION=${K8S_VERSION:-1.15.12}
# K8S_VERSION=${K8S_VERSION:-1.16.15}
# K8S_VERSION=${K8S_VERSION:-1.17.14}
# K8S_VERSION=${K8S_VERSION:-1.18.12}
# K8S_VERSION=${K8S_VERSION:-1.19.7}
K8S_VERSION=${K8S_VERSION:-1.20.2}
export VALIDATE_YAML=true
killall kubectl || true
wget -c --progress=bar:force:noscroll -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/linux/amd64/kubectl
chmod +x /usr/local/bin/kubectl
usermod -a -G docker vagrant
mkdir -p /home/vagrant/.minikube
ln -svf /home/vagrant/.minikube /root/.minikube
mkdir -p /home/vagrant/.kube
ln -svf /home/vagrant/.kube /root/.kube
chown vagrant:vagrant -R /home/vagrant/
sudo -H -u vagrant minikube delete
sudo -H -u vagrant minikube config set memory 5G
sudo -H -u vagrant minikube config set driver docker
sudo -H -u vagrant minikube config set kubernetes-version ${K8S_VERSION}
sudo -H -u vagrant minikube start
sudo -H -u vagrant minikube addons enable ingress
sudo -H -u vagrant minikube addons enable ingress-dns
sudo -H -u vagrant minikube addons enable metrics-server
# minikube config set vm-driver none
# minikube config set kubernetes-version ${K8S_VERSION}
# minikube start --vm=true
# minikube addons enable ingress
# minikube addons enable ingress-dns
# minikube addons enable metrics-server
#krew
(
curl -fsSLO "https://github.com/kubernetes-sigs/krew/releases/latest/download/krew.tar.gz" &&
tar zxvf krew.tar.gz &&
KREW=./krew-"$(uname | tr '[:upper:]' '[:lower:]')_amd64" &&
sudo -H -u vagrant "$KREW" install krew
)
sudo -H -u vagrant bash -c 'echo export PATH="\${KREW_ROOT:-\$HOME/.krew}/bin:\$PATH" | tee \$HOME/.bashrc'
echo export PATH="/home/vagrant/.krew/bin:$PATH" | tee $HOME/.bashrc
source $HOME/.bashrc
export KREW_ROOT=/home/vagrant/.krew
kubectl krew install tap
kubectl krew install debug
kubectl krew install sniff
kubectl krew install flame
cd /vagrant/
git_branch=$(git rev-parse --abbrev-ref HEAD)
export OPERATOR_RELEASE=$(cat release)
export BRANCH=${BRANCH:-$git_branch}
export OPERATOR_NAMESPACE=${OPERATOR_NAMESPACE:-kube-system}
export OPERATOR_IMAGE=altinity/clickhouse-operator:${OPERATOR_RELEASE}
export METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:${OPERATOR_RELEASE}
if ! kubectl get deployment clickhouse-operator -n "${OPERATOR_NAMESPACE}" 1>/dev/null 2>/dev/null; then
cd /vagrant/deploy/operator/
bash -x ./clickhouse-operator-install.sh
cd /vagrant
fi
export PROMETHEUS_NAMESPACE=${PROMETHEUS_NAMESPACE:-prometheus}
cd /vagrant/deploy/prometheus/
kubectl delete ns ${PROMETHEUS_NAMESPACE} || true
bash -xe ./create-prometheus.sh
cd /vagrant/
export GRAFANA_NAMESPACE=${GRAFANA_NAMESPACE:-grafana}
cd /vagrant/deploy/grafana/grafana-with-grafana-operator/
kubectl delete ns ${GRAFANA_NAMESPACE} || true
bash -xe ./install-grafana-operator.sh
bash -xe ./install-grafana-with-operator.sh
cd /vagrant
echo "Wait when clickhouse operator installation finished"
while [[ $(kubectl get pods --all-namespaces -l app=clickhouse-operator | wc -l) != "2" ]]; do
printf "."
sleep 1
done
echo "...DONE"
# open http://localhost:9090/targets and check clickhouse-monitor is exists
kubectl --namespace="${PROMETHEUS_NAMESPACE}" port-forward --address 0.0.0.0 service/prometheus 9090 </dev/null &>/dev/null &
# open http://localhost:9093/alerts and check which alerts is exists
kubectl --namespace="${PROMETHEUS_NAMESPACE}" port-forward --address 0.0.0.0 service/alertmanager 9093 </dev/null &>/dev/null &
# open http://localhost:3000/ and check prometheus datasource exists and grafana dashboard exists
kubectl --namespace="${GRAFANA_NAMESPACE}" port-forward --address 0.0.0.0 service/grafana-service 3000 </dev/null &>/dev/null &
# open http://localhost:8888/chi and check exists clickhouse installations
kubectl --namespace="${OPERATOR_NAMESPACE}" port-forward --address 0.0.0.0 service/clickhouse-operator-metrics 8888 </dev/null &>/dev/null &
for image in $(cat ./tests/configs/test-017-multi-version.yaml | yq r - "spec.templates.podTemplates[*].spec.containers[*].image"); do
docker pull ${image}
done
pip3 install -r /vagrant/tests/requirements.txt
python3 /vagrant/tests/test.py --only=operator/*
python3 /vagrant/tests/test_examples.py
python3 /vagrant/tests/test_metrics_exporter.py
python3 /vagrant/tests/test_metrics_alerts.py
SHELL
end