From db770e21dc19056e58b77e29d96faa501e02d592 Mon Sep 17 00:00:00 2001 From: Yellowman Date: Mon, 1 Apr 2019 10:37:02 +0200 Subject: [PATCH 1/3] :sparkles: add cron on gcp --- cron/insert/insert.go | 2 +- gcp/cron/cronjob-bruteforce.yaml | 49 ++++++++++++++++++++++++++++++++ gcp/cron/cronjob-insert.yml | 49 ++++++++++++++++++++++++++++++++ gcp/cron/cronjob-select.yml | 48 +++++++++++++++++++++++++++++++ gcp/cron/cronjob-temporary.yml | 48 +++++++++++++++++++++++++++++++ gcp/deployment/api.yml | 4 +-- gcp/deployment/front.yml | 2 +- 7 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 gcp/cron/cronjob-bruteforce.yaml create mode 100644 gcp/cron/cronjob-insert.yml create mode 100644 gcp/cron/cronjob-select.yml create mode 100644 gcp/cron/cronjob-temporary.yml diff --git a/cron/insert/insert.go b/cron/insert/insert.go index 56e4f13..48f476c 100644 --- a/cron/insert/insert.go +++ b/cron/insert/insert.go @@ -25,7 +25,7 @@ func main() { } _, execErr := stmt.Exec( - gofakeit.Name(), + gofakeit.BeerName(), 10.0, gofakeit.Name(), gofakeit.NameSuffix(), diff --git a/gcp/cron/cronjob-bruteforce.yaml b/gcp/cron/cronjob-bruteforce.yaml new file mode 100644 index 0000000..8da2e29 --- /dev/null +++ b/gcp/cron/cronjob-bruteforce.yaml @@ -0,0 +1,49 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: bruteforce +spec: + schedule: "*/15 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: bruteforce + image: gcr.io/kubernetes-demo-232217/sesame_cron:v1 + imagePullPolicy: IfNotPresent + env: + - name: MYSQL_HOST + value: 127.0.0.1 + - name: MYSQL_USER + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: username + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: password + args: + - go + - run + - bruteforce/bruteforce.go + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.13 + command: ["/cloud_sql_proxy", + "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", + "-credential_file=/secrets/cloudsql/credentials.json", + "& pid=$! $$ (sleep 45 && kill -9 $pid 2>/dev/null)"] + securityContext: + runAsUser: 2 # non-root user + allowPrivilegeEscalation: false + volumeMounts: + - name: cloudsql-instance-credentials + mountPath: /secrets/cloudsql + readOnly: true + volumes: + - name: cloudsql-instance-credentials + secret: + secretName: cloudsql-instance-credentials + restartPolicy: OnFailure \ No newline at end of file diff --git a/gcp/cron/cronjob-insert.yml b/gcp/cron/cronjob-insert.yml new file mode 100644 index 0000000..a97cbb8 --- /dev/null +++ b/gcp/cron/cronjob-insert.yml @@ -0,0 +1,49 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: insertbobba +spec: + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: insertbobba + image: gcr.io/kubernetes-demo-232217/sesame_cron:v1 + imagePullPolicy: IfNotPresent + env: + - name: MYSQL_HOST + value: 127.0.0.1 + - name: MYSQL_USER + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: username + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: password + args: + - go + - run + - insert/insert.go + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.13 + command: ["/cloud_sql_proxy", + "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", + "-credential_file=/secrets/cloudsql/credentials.json", + "& pid=$! $$ (sleep 45 && kill -9 $pid 2>/dev/null)"] + securityContext: + runAsUser: 2 # non-root user + allowPrivilegeEscalation: false + volumeMounts: + - name: cloudsql-instance-credentials + mountPath: /secrets/cloudsql + readOnly: true + volumes: + - name: cloudsql-instance-credentials + secret: + secretName: cloudsql-instance-credentials + restartPolicy: OnFailure diff --git a/gcp/cron/cronjob-select.yml b/gcp/cron/cronjob-select.yml new file mode 100644 index 0000000..efd2c35 --- /dev/null +++ b/gcp/cron/cronjob-select.yml @@ -0,0 +1,48 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: select +spec: + schedule: "*/30 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: select + image: gcr.io/kubernetes-demo-232217/sesame_cron:v1 + imagePullPolicy: IfNotPresent + env: + - name: MYSQL_HOST + value: 127.0.0.1 + - name: MYSQL_USER + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: username + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: password + args: + - go + - run + - select/select.go + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.13 + command: ["/cloud_sql_proxy", + "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", + "-credential_file=/secrets/cloudsql/credentials.json"] + securityContext: + runAsUser: 2 # non-root user + allowPrivilegeEscalation: false + volumeMounts: + - name: cloudsql-instance-credentials + mountPath: /secrets/cloudsql + readOnly: true + volumes: + - name: cloudsql-instance-credentials + secret: + secretName: cloudsql-instance-credentials + restartPolicy: OnFailure diff --git a/gcp/cron/cronjob-temporary.yml b/gcp/cron/cronjob-temporary.yml new file mode 100644 index 0000000..bcda845 --- /dev/null +++ b/gcp/cron/cronjob-temporary.yml @@ -0,0 +1,48 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: temporary +spec: + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: temporary + image: gcr.io/kubernetes-demo-232217/sesame_cron:v1 + imagePullPolicy: IfNotPresent + env: + - name: MYSQL_HOST + value: 127.0.0.1 + - name: MYSQL_USER + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: username + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: password + args: + - go + - run + - temporary/temporary.go + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.13 + command: ["/cloud_sql_proxy", + "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", + "-credential_file=/secrets/cloudsql/credentials.json"] + securityContext: + runAsUser: 2 # non-root user + allowPrivilegeEscalation: false + volumeMounts: + - name: cloudsql-instance-credentials + mountPath: /secrets/cloudsql + readOnly: true + volumes: + - name: cloudsql-instance-credentials + secret: + secretName: cloudsql-instance-credentials + restartPolicy: OnFailure diff --git a/gcp/deployment/api.yml b/gcp/deployment/api.yml index f040b15..43480c1 100644 --- a/gcp/deployment/api.yml +++ b/gcp/deployment/api.yml @@ -24,7 +24,7 @@ metadata: app: bobba-api tier: backend spec: - replicas: 2 + replicas: 9 selector: matchLabels: app: bobba-api @@ -59,7 +59,7 @@ spec: ports: - containerPort: 8000 - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.11 + image: gcr.io/cloudsql-docker/gce-proxy:1.13 command: ["/cloud_sql_proxy", "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", "-credential_file=/secrets/cloudsql/credentials.json"] diff --git a/gcp/deployment/front.yml b/gcp/deployment/front.yml index 0579aa5..047251b 100644 --- a/gcp/deployment/front.yml +++ b/gcp/deployment/front.yml @@ -20,7 +20,7 @@ metadata: app: bobba-vue tier: frontend spec: - replicas: 2 + replicas: 9 selector: matchLabels: app: bobba-vue From 600955a8b0c274d62f8045ca814201f464a7cbab Mon Sep 17 00:00:00 2001 From: Yellowman Date: Mon, 1 Apr 2019 17:50:37 +0200 Subject: [PATCH 2/3] :sparkles: add cloudproxy sql deployment & update cron --- gcp/cron/cronjob-bruteforce.yaml | 19 +----------- gcp/cron/cronjob-insert.yml | 21 ++----------- gcp/cron/cronjob-select.yml | 18 +---------- gcp/cron/cronjob-temporary.yml | 18 +---------- gcp/deployment/cloudsql.yml | 53 ++++++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 71 deletions(-) create mode 100644 gcp/deployment/cloudsql.yml diff --git a/gcp/cron/cronjob-bruteforce.yaml b/gcp/cron/cronjob-bruteforce.yaml index 8da2e29..1ee84f1 100644 --- a/gcp/cron/cronjob-bruteforce.yaml +++ b/gcp/cron/cronjob-bruteforce.yaml @@ -14,7 +14,7 @@ spec: imagePullPolicy: IfNotPresent env: - name: MYSQL_HOST - value: 127.0.0.1 + value: cloudsql-proxy-svc - name: MYSQL_USER valueFrom: secretKeyRef: @@ -29,21 +29,4 @@ spec: - go - run - bruteforce/bruteforce.go - - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.13 - command: ["/cloud_sql_proxy", - "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", - "-credential_file=/secrets/cloudsql/credentials.json", - "& pid=$! $$ (sleep 45 && kill -9 $pid 2>/dev/null)"] - securityContext: - runAsUser: 2 # non-root user - allowPrivilegeEscalation: false - volumeMounts: - - name: cloudsql-instance-credentials - mountPath: /secrets/cloudsql - readOnly: true - volumes: - - name: cloudsql-instance-credentials - secret: - secretName: cloudsql-instance-credentials restartPolicy: OnFailure \ No newline at end of file diff --git a/gcp/cron/cronjob-insert.yml b/gcp/cron/cronjob-insert.yml index a97cbb8..4b7d286 100644 --- a/gcp/cron/cronjob-insert.yml +++ b/gcp/cron/cronjob-insert.yml @@ -10,11 +10,11 @@ spec: spec: containers: - name: insertbobba - image: gcr.io/kubernetes-demo-232217/sesame_cron:v1 + image: gcr.io/kubernetes-demo-232217/sesame_cron:v1.1 imagePullPolicy: IfNotPresent env: - name: MYSQL_HOST - value: 127.0.0.1 + value: cloudsql-proxy-svc - name: MYSQL_USER valueFrom: secretKeyRef: @@ -29,21 +29,4 @@ spec: - go - run - insert/insert.go - - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.13 - command: ["/cloud_sql_proxy", - "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", - "-credential_file=/secrets/cloudsql/credentials.json", - "& pid=$! $$ (sleep 45 && kill -9 $pid 2>/dev/null)"] - securityContext: - runAsUser: 2 # non-root user - allowPrivilegeEscalation: false - volumeMounts: - - name: cloudsql-instance-credentials - mountPath: /secrets/cloudsql - readOnly: true - volumes: - - name: cloudsql-instance-credentials - secret: - secretName: cloudsql-instance-credentials restartPolicy: OnFailure diff --git a/gcp/cron/cronjob-select.yml b/gcp/cron/cronjob-select.yml index efd2c35..350fbb5 100644 --- a/gcp/cron/cronjob-select.yml +++ b/gcp/cron/cronjob-select.yml @@ -14,7 +14,7 @@ spec: imagePullPolicy: IfNotPresent env: - name: MYSQL_HOST - value: 127.0.0.1 + value: cloudsql-proxy-svc - name: MYSQL_USER valueFrom: secretKeyRef: @@ -29,20 +29,4 @@ spec: - go - run - select/select.go - - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.13 - command: ["/cloud_sql_proxy", - "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", - "-credential_file=/secrets/cloudsql/credentials.json"] - securityContext: - runAsUser: 2 # non-root user - allowPrivilegeEscalation: false - volumeMounts: - - name: cloudsql-instance-credentials - mountPath: /secrets/cloudsql - readOnly: true - volumes: - - name: cloudsql-instance-credentials - secret: - secretName: cloudsql-instance-credentials restartPolicy: OnFailure diff --git a/gcp/cron/cronjob-temporary.yml b/gcp/cron/cronjob-temporary.yml index bcda845..eda530f 100644 --- a/gcp/cron/cronjob-temporary.yml +++ b/gcp/cron/cronjob-temporary.yml @@ -14,7 +14,7 @@ spec: imagePullPolicy: IfNotPresent env: - name: MYSQL_HOST - value: 127.0.0.1 + value: cloudsql-proxy-svc - name: MYSQL_USER valueFrom: secretKeyRef: @@ -29,20 +29,4 @@ spec: - go - run - temporary/temporary.go - - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.13 - command: ["/cloud_sql_proxy", - "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", - "-credential_file=/secrets/cloudsql/credentials.json"] - securityContext: - runAsUser: 2 # non-root user - allowPrivilegeEscalation: false - volumeMounts: - - name: cloudsql-instance-credentials - mountPath: /secrets/cloudsql - readOnly: true - volumes: - - name: cloudsql-instance-credentials - secret: - secretName: cloudsql-instance-credentials restartPolicy: OnFailure diff --git a/gcp/deployment/cloudsql.yml b/gcp/deployment/cloudsql.yml new file mode 100644 index 0000000..3f33c61 --- /dev/null +++ b/gcp/deployment/cloudsql.yml @@ -0,0 +1,53 @@ +kind: Service +apiVersion: v1 +metadata: + name: cloudsql-proxy-svc +spec: + clusterIP: None + selector: + app: cloudsql + tier: proxy + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudsql + labels: + app: cloudsql + tier: proxy +spec: + replicas: 1 + selector: + matchLabels: + app: cloudsql + tier: proxy + template: + metadata: + labels: + app: cloudsql + tier: proxy + spec: + containers: + - name: cloudsql-proxy + #image: gcr.io/kubernetes-demo-232217/miniserver:v1.2 + image: gcr.io/cloudsql-docker/gce-proxy:1.13 + ports: + - containerPort: 3306 + command: ["/cloud_sql_proxy", + "-instances=kubernetes-demo-232217:us-central1:db=tcp:0.0.0.0:3306", + "-credential_file=/secrets/cloudsql/credentials.json"] + securityContext: + runAsUser: 2 # non-root user + allowPrivilegeEscalation: false + volumeMounts: + - name: cloudsql-instance-credentials + mountPath: /secrets/cloudsql + readOnly: true + volumes: + - name: cloudsql-instance-credentials + secret: + secretName: cloudsql-instance-credentials From c92c4204744846e3a07aadb8d7f5285322e0b354 Mon Sep 17 00:00:00 2001 From: Yellowman Date: Tue, 2 Apr 2019 11:29:26 +0200 Subject: [PATCH 3/3] :memo: add docs regarding deploying cronjob --- bobba-helm-chart/templates/sql-proxy.yaml | 2 +- docs/_data/navigation.yml | 2 + docs/gcp/cronjob.md | 124 ++++++++++++++++++++++ docs/index.md | 1 + gcp/deployment/cloudsql.yml | 3 +- 5 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 docs/gcp/cronjob.md diff --git a/bobba-helm-chart/templates/sql-proxy.yaml b/bobba-helm-chart/templates/sql-proxy.yaml index 1bad498..a08a523 100644 --- a/bobba-helm-chart/templates/sql-proxy.yaml +++ b/bobba-helm-chart/templates/sql-proxy.yaml @@ -1,6 +1,6 @@ {{- define "bobba-helm-chart.sql-proxy" -}} - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.11 + image: gcr.io/cloudsql-docker/gce-proxy:1.13 command: ["/cloud_sql_proxy", "-instances=kubernetes-demo-232217:us-central1:db=tcp:3306", "-credential_file=/secrets/cloudsql/credentials.json"] diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml index fea34ca..7f26b27 100644 --- a/docs/_data/navigation.yml +++ b/docs/_data/navigation.yml @@ -58,6 +58,8 @@ docs: url: /gcp/deployment_front - title: Load balance multiple cluster url: /gcp/mci + - title: Deploying CronJob + url: /gcp/cronjob - title: Debug your instance url: /gcp/debug diff --git a/docs/gcp/cronjob.md b/docs/gcp/cronjob.md new file mode 100644 index 0000000..8133217 --- /dev/null +++ b/docs/gcp/cronjob.md @@ -0,0 +1,124 @@ +--- +layout: single +classes: wide +title: Adding cronjob to your cluster with cloudsql proxy +sidebar: + nav: "docs" +--- + +Since we deployed cronjob on our Minikube cluster. You might be tempt to deploy cronjob to your GKE cluster. As we use the cloudsql-proxy to deploy our API it might be a good idea to use the same kind of deployment. Having a pod which is containing 2 containers. The cronjob & the cloudsql-proxy pod as a sidecar. + +## Issues + +However this solution has an issue. Indeed the job doesn't really support the sidecar feature yet. This mean that once your job is completed the job won't terminate the sidecar container which lead to the fact that the job's pod got never delete... + +At this day (02/04/2019) this feature is currently being under consideration. See this issue [issue](https://github.com/kubernetes/kubernetes/issues/25908) + +## Solution + +In order to use the full capabilities of the cloudsql-proxy feature we will use the strategy list below: + +> Note: There are other kind of strategy that you could use... but this one seems to me the cleanest way to do. + +This strategy consist of the following stpes: +- Deploying a ```single pod``` which run the cloudsql-proxy. +- Deploying a service which is not exposed outside of the cluster we'll only exposed it through the kube-dns (Kubernetes use the service name as the domain) +- CronJob will target the DNS instead of the localhost + +> ⚠️ Warning: This solution has one drawback. This single pod will be on a node. As you remember node are mortal thus it might be possible that a cronjob might not be able to communicate with the cloudsql-proxy pod as the node has been killed or is being recreated. + +## Implementation + +### Deployment + +Deploying the cloudsql-proxy deployment is pretty straightforward. Create a deployment yml file which you register your cloudsql-proxy container and expose the pod to ```3306```. An already ```cloudsql.yml``` exist in the ```gcp/deployment/cloudsql.yml``` + +A subtle difference should be note: + +```yaml +command: ["/cloud_sql_proxy", + "-instances=::=tcp:0.0.0.0:3306", + "-credential_file=/secrets/cloudsql/credentials.json"] +``` + +⚠️ In the command value we're exposing the database by bypassing the firewall which should be take with precaution. + +### Service + +In our case we only need access to the database from the cluster. Thus we don't need to expose the database outside of the cluster (i.e NodePort, Loadbalancer). + +Furthermore for security purposes it's not a good idea to expose the pod outside of the world. Thus we're only going to expose our deployment to the kube-dns by creating a ```Headless service```. Below is the configuration which could be found here: ```gcp/deployment/cloudsql.yml``` + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: cloudsql-proxy-svc +spec: + clusterIP: None + selector: + app: cloudsql + tier: proxy + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 +``` + +By using clusterIP to none we're only referencing our service to the kube-dns. In return we can communicate to the deployment with the ```name``` of the deployment. For more details on headless service please check the [documentation](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) + +## Configure our cron + +Like for our ```bobba-api``` deployment we're going to apply the same configuration to our cronjob with one difference. The configuration is already applicated to the cronjob files available in this folder ```gcp/cron/*.yml``` + +```yaml +env: + - name: MYSQL_HOST + value: cloudsql-proxy-svc + - name: MYSQL_USER + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: username + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: cloudsql-db-credentials + key: password +``` + +Instead of passing the ```127.0.0.1``` IP address to the ```MYSQL_HOST``` we're using the ```DNS discovery``` system in order to communicate with our database. + +## Let's deploy + +As we have every required files let's deploy. First go to the ```gcp/deployment``` folder & run this command + +```shell +kubectl apply -f cloudsql.yml +``` + +*Make sure that the pod is running before processing to the next steps* + +Now let's deploy the cron. Move to the folder ```gcp/cron``` and run this command + +```shell +kubectl apply -f . +``` + +Make sure that the cronjob has been deployed to your cluster by running this command + +```shell +kubectl get cronjob +``` + +... Wait for around 10 minutes ⏰ + +And you should see a cron pod running ! (this should be the insertbobba pod) + +If the job run successfully you've just learn how to deploy cronjob with cloudsql-proxy to your GKE cluster. + +## Resources + +[Headless service in Kubernetes](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) + +[Running CloudSQL Proxy as a standalone](https://github.com/GoogleCloudPlatform/kubernetes-engine-samples/issues/8) diff --git a/docs/index.md b/docs/index.md index a1ec07b..49982de 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,6 +41,7 @@ Throughout this guide we'll learn several thing which are describe in the table * * [Configure an Ingress resources](gcp/ingress.md) * * [Build the Front docker image](gcp/front.md) * * [Deploy the front image](gcp/deployment_front.md) +* * [Deploying cronjob](gcp/cronjob.md) * * [Load balance multiple cluster](gcp/mci.md) * * [Horizontal autoscaler](k8s/hscaler.md) * * [Debugging](gcp/debug.md) diff --git a/gcp/deployment/cloudsql.yml b/gcp/deployment/cloudsql.yml index 3f33c61..f1e8795 100644 --- a/gcp/deployment/cloudsql.yml +++ b/gcp/deployment/cloudsql.yml @@ -33,12 +33,11 @@ spec: spec: containers: - name: cloudsql-proxy - #image: gcr.io/kubernetes-demo-232217/miniserver:v1.2 image: gcr.io/cloudsql-docker/gce-proxy:1.13 ports: - containerPort: 3306 command: ["/cloud_sql_proxy", - "-instances=kubernetes-demo-232217:us-central1:db=tcp:0.0.0.0:3306", + "-instances=::=tcp:0.0.0.0:3306", "-credential_file=/secrets/cloudsql/credentials.json"] securityContext: runAsUser: 2 # non-root user