feat(cicd): Improve terraform and finish gateway

* Improve terraform artifacts * Try to implement native vpc cluster * use native vpc cluster fix the gateway * Finish gateway * update readme * ubncomment gateway tetraform * improve comments
CharlyJazz · Dec 10, 2023 · 9c98fd8 · 9c98fd8
1 parent 09681da
commit 9c98fd8
Show file tree

Hide file tree

Showing 24 changed files with 163 additions and 62 deletions.
diff --git a/.github/workflows/aggregated-messages-consumer-deploy.yml b/.github/workflows/aggregated-messages-consumer-deploy.yml
@@ -27,7 +27,7 @@ jobs:
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v1'
+        uses: 'google-github-actions/auth@v2'
         with:
           credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 

diff --git a/.github/workflows/auth-microservice-deploy.yml b/.github/workflows/auth-microservice-deploy.yml
@@ -27,7 +27,7 @@ jobs:
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v1'
+        uses: 'google-github-actions/auth@v2'
         with:
           credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 

diff --git a/.github/workflows/camel-microservice-deploy.yml b/.github/workflows/camel-microservice-deploy.yml
@@ -27,7 +27,7 @@ jobs:
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v1'
+        uses: 'google-github-actions/auth@v2'
         with:
           credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 

diff --git a/.github/workflows/chat-microservice-deploy.yml b/.github/workflows/chat-microservice-deploy.yml
@@ -27,7 +27,7 @@ jobs:
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v1'
+        uses: 'google-github-actions/auth@v2'
         with:
           credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 

diff --git a/.github/workflows/frontend-deploy.yml b/.github/workflows/frontend-deploy.yml
@@ -27,7 +27,7 @@ jobs:
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v1'
+        uses: 'google-github-actions/auth@v2'
         with:
           credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 

diff --git a/.gitignore b/.gitignore
@@ -44,4 +44,6 @@ service_account_github_action.json
 
 firestore_service_account.json
 
-env.prod
+env.prod
+
+firestore_service_acount_chat_messages.json
diff --git a/README.md b/README.md
@@ -212,7 +212,26 @@ NEXT_PUBLIC_MESSAGING_SENDER_ID="..."
 NEXT_PUBLIC_APP_ID="..."
 ```
 
----
+### Terraform
+
+- Create project
+- `gcloud auth application-default set-quota-project $PROJECT_NAME`
+- `gcloud config set project $PROJECT_NAME`
+- `terraform init`
+- Comment `gke_secrets.tf` and `kafka.tf`
+- `terraform apply`
+- `gcloud container clusters get-credentials chat2-405718-gke --region us-central1`
+- Set `gke_config_context` in `terraform.tfvars`
+- Uncomment `gke_secrets.tf` and `kafka.tf`
+- `terraform apply`
+- ./apply_k8s_configs.sh
+- Set in a .env `NEXT_PUBLIC_AUTH_MICROSERVICE` and `NEXT_PUBLIC_CHAT_MICROSERVICE` using the load balancing external IP
+- Go to firebase and create the project and a web app for this gcp project and fill the .env
+- Run ` kubectl create secret generic frontend-secrets --from-env-file=.env`
+- https://github.com/google-github-actions/auth/blob/main/docs/EXAMPLES.md to set GOOGLE_CREDENTIALS secret
+- GKE_SA_KEY gcloud iam service-accounts keys create key.json --iam-account=SA_EMAIL
+- export GKE_SA_KEY=$(cat key.json | base64)
+- use key.json in secret GKE_SA_KEY
 
 ### License
 

diff --git a/Terraform.md b/Terraform.md
@@ -5,11 +5,11 @@
 - Install gcloud and login
 - https://github.com/hashicorp/learn-terraform-provision-gke-cluster
 - https://developer.hashicorp.com/terraform/tutorials/kubernetes/gke
-- Set quota `gcloud auth application-default set-quota-project apache-camel-chat-development`
+- Set quota `gcloud auth application-default set-quota-project chat2-405718`
 - terraform init - Save variables
 - terraform apply - Create resources in GCP
-- Super importante correr esto diario gcloud config set project
-- gcloud container clusters get-credentials chat1-405416-gke --region us-central1 - kubectl context
+- Super importante correr esto diario gcloud config set project chat2-405718
+- gcloud container clusters get-credentials chat2-405718-gke --region us-central1 - kubectl context
 
 ### Configure kubectl
 
@@ -24,16 +24,6 @@
 
 - https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/kubectl_filename_list#matches
 
-### Save State in GCP Storage (TODO)
-
-- gcloud services enable storage.googleapis.com
-- I created a role for my user with all the storage permissions needed
-- gcloud iam service-accounts create terraform-admin \
-   --description="Service account for Terraform administration" \
-   --display-name="Terraform Admin" \
-   --project=chat1-405416
-- gcloud auth application-default login --project chat1-405416 - https://fabianlee.org/2023/06/11/terraform-fixing-error-querying-cloud-storage-failed-storage-bucket-doesnt-exist/
-
 ### Create Github Action Pipeline for auth-microservice
 
 - gcloud iam service-accounts create github-action

diff --git a/api-gateway/gateway.yaml b/api-gateway/gateway.yaml
@@ -3,7 +3,7 @@ kind: Gateway
 metadata:
   name: gke-l7-gxlb-api-gateway
 spec:
-  gatewayClassName: gke-l7-gxlb # Probar este
+  gatewayClassName: gke-l7-gxlb
   listeners:
   - name: http-listener
     protocol: HTTP

diff --git a/apply_k8s_configs.sh b/apply_k8s_configs.sh
@@ -5,16 +5,17 @@ set -e
 
 # List of directories containing Kubernetes configurations
 declare -a k8s_dirs=(
-    "kafka/k8s"
-    "chat-microservice/k8s"
-    "apache-camel-service-bus/k8s"
-    "auth-microservice/k8s"
-    "aggregated-messages-consumer/k8s"
-    "frontend-web/k8s"
+    # "kafka/k8s"
+    api-gateway
+    # "chat-microservice/k8s"
+    # "apache-camel-service-bus/k8s"
+    # "auth-microservice/k8s"
+    # "aggregated-messages-consumer/k8s"
+    # "frontend-web/k8s"
 )
 
-kubectl create secret generic firebase-secret --from-file=service_account_key.json=./aggregated-messages-consumer/service_account_key.json
-kubectl create secret generic frontend-secrets --from-env-file=.env
+# kubectl create secret generic firebase-secret --from-file=service_account_key.json=./aggregated-messages-consumer/service_account_key.json
+# kubectl create secret generic frontend-secrets --from-env-file=.env
 
 # Function to apply Kubernetes configurations
 apply_configs() {

diff --git a/auth-microservice/app/api/__init__.py b/auth-microservice/app/api/__init__.py
@@ -3,6 +3,6 @@
 from app.api.health import router as health_router
 from app.api.v1 import router as v1_router
 
-router = APIRouter(prefix="/api")
+router = APIRouter(prefix="/auth/api")
 router.include_router(v1_router)
 router.include_router(health_router)
diff --git a/chat-microservice/app/api/__init__.py b/chat-microservice/app/api/__init__.py
@@ -3,6 +3,6 @@
 from app.api.health import router as health_router
 from app.api.v1 import router as v1_router
 
-router = APIRouter(prefix="/api")
+router = APIRouter(prefix="/chat/api")
 router.include_router(v1_router)
 router.include_router(health_router)
diff --git a/chat-microservice/app/api/health.py b/chat-microservice/app/api/health.py
@@ -10,6 +10,4 @@
 
 @router.get("/", status_code=200)
 async def health():
-    # TODO: VALIDATE CONFIGURATION SCHEMA REQUIRED VALUES
-    # TODO: CHECK CASSANDRA CONNECTION, KAFKA CONNECTION
     return Response(status_code=200)
diff --git a/terraform/api_gateway.tf b/terraform/api_gateway.tf
@@ -0,0 +1,16 @@
+
+data "kubectl_filename_list" "api_gateway_manifests" {
+  pattern = "../api-gateway/*.yaml"
+}
+
+resource "kubectl_manifest" "api_gateway" {
+  count     = length(data.kubectl_filename_list.api_gateway_manifests.matches)
+  yaml_body = file(element(data.kubectl_filename_list.api_gateway_manifests.matches, count.index))
+
+  depends_on = [
+    google_container_cluster.primary,
+    google_compute_network.vpc,
+    google_compute_subnetwork.subnet,
+    google_compute_subnetwork.managed_proxy_subnet,
+  ]
+}
diff --git a/terraform/backend.tf b/terraform/backend.tf
@@ -1,8 +1,6 @@
 terraform {
   backend "gcs" {
-    bucket = "state-bucket-tfstate-chat1"
+    bucket = "state-bucket-tfstate-chat3"
     prefix = "terraform/state"
-    # credentials = "/Users/carlosazuaje/.config/gcloud/application_default_credentials.json"
-    # impersonate_service_account = "terraform-admin@chat1-405416.iam.gserviceaccount.com"
   }
 }
diff --git a/terraform/bucket.tf b/terraform/bucket.tf
@@ -1,5 +1,10 @@
 resource "google_storage_bucket" "default" {
-  name          = "state-bucket-tfstate-chat1"
+  # Only comment the count attribute if
+  # you did not create the bucket manually in GCP
+  # In that case you should comment the backend config
+  # for the first apply
+  count         = 0
+  name          = "state-bucket-tfstate-chat3"
   force_destroy = false
   location      = "US"
   storage_class = "STANDARD"

diff --git a/terraform/firebase.tf b/terraform/firebase.tf
@@ -29,19 +29,8 @@ resource "google_service_account_key" "firestore_service_account_key" {
   public_key_type    = "TYPE_X509_PEM_FILE"
 }
 
-output "firestore_service_account_json" {
-  value = jsonencode({
-    type                        = "service_account",
-    project_id                  = var.project_id,
-    private_key_id              = google_service_account_key.firestore_service_account_key.name,
-    private_key                 = base64decode(google_service_account_key.firestore_service_account_key.private_key),
-    client_email                = google_service_account.firestore_service_account.email,
-    client_id                   = google_service_account.firestore_service_account.name,
-    auth_uri                    = "https://accounts.google.com/o/oauth2/auth",
-    token_uri                   = "https://oauth2.googleapis.com/token",
-    auth_provider_x509_cert_url = "https://www.googleapis.com/oauth2/v1/certs",
-    client_x509_cert_url        = "https://www.googleapis.com/robot/v1/metadata/x509/${google_service_account.firestore_service_account.email}"
-  })
-  sensitive = true
-}
 
+resource "local_file" "firestore_service_acount_chat_messages" {
+  filename = "./firestore_service_acount_chat_messages.json"
+  content  = base64decode(google_service_account_key.firestore_service_account_key.private_key)
+}
diff --git a/terraform/gke.tf b/terraform/gke.tf
@@ -29,9 +29,13 @@ resource "google_container_cluster" "primary" {
   remove_default_node_pool = true
   initial_node_count       = 1
 
-  network    = google_compute_network.vpc.name
-  subnetwork = google_compute_subnetwork.subnet.name
+  network    = google_compute_network.vpc.id
+  subnetwork = google_compute_subnetwork.subnet.id
 
+  ip_allocation_policy {
+    cluster_secondary_range_name  = "pod-ranges"
+    services_secondary_range_name = google_compute_subnetwork.subnet.secondary_ip_range.0.range_name
+  }
   gateway_api_config {
     channel = "CHANNEL_STANDARD"
   }

diff --git a/terraform/gke_secrets.tf b/terraform/gke_secrets.tf
@@ -6,8 +6,10 @@ resource "kubernetes_secret" "firebase_secret" {
   }
 
   data = {
-    "service_account_key.json" = file("${path.module}/firestore_service_account.json")
+    "service_account_key.json" = file("${path.module}/firestore_service_acount_chat_messages.json")
   }
+
+  depends_on = [local_file.firestore_service_acount_chat_messages]
 }
 
 # kubectl create secret generic frontend-secrets --from-env-file=.env

diff --git a/terraform/outputs.tf b/terraform/outputs.tf
@@ -6,6 +6,14 @@ variable "region" {
   description = "region"
 }
 
+variable "gke_config_context" {
+  description = "GKE Cluster name. Set it right after first terraform apply."
+}
+
+variable "managed_proxy_subnet" {
+  description = "Name fo the proxy subnet for the api gateway"
+}
+
 output "region" {
   value       = var.region
   description = "GCloud Region"

diff --git a/terraform/provider.tf b/terraform/provider.tf
@@ -10,5 +10,5 @@ provider "kubectl" {
 
 provider "kubernetes" {
   config_path    = "~/.kube/config"
-  config_context = "gke_chat1-405416_us-central1_chat1-405416-gke"
+  config_context = var.gke_config_context
 }
diff --git a/terraform/service_account_github.tf b/terraform/service_account_github.tf
@@ -0,0 +1,36 @@
+resource "google_service_account" "github_action" {
+  account_id   = "github-action"
+  display_name = "GitHub Action Service Account"
+}
+
+resource "google_project_iam_binding" "container_admin" {
+  project = var.project_id
+  role    = "roles/container.admin"
+
+  members = [
+    "serviceAccount:${google_service_account.github_action.email}",
+  ]
+
+  depends_on = [google_service_account.github_action]
+}
+
+resource "google_project_iam_binding" "storage_admin" {
+  project = var.project_id
+  role    = "roles/storage.admin"
+
+  members = [
+    "serviceAccount:${google_service_account.github_action.email}",
+  ]
+  depends_on = [google_service_account.github_action]
+}
+
+resource "google_project_iam_binding" "cluster_viewer" {
+  project = var.project_id
+  role    = "roles/container.clusterViewer"
+
+  members = [
+    "serviceAccount:${google_service_account.github_action.email}",
+  ]
+
+  depends_on = [google_service_account.github_action]
+}
diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars
@@ -1,2 +1,4 @@
-project_id = "chat1-405416"
-region     = "us-central1"
+project_id           = "chat3-407706"
+region               = "us-central1"
+gke_config_context   = "gke_chat3-407706_us-central1_chat3-407706-gke"
+managed_proxy_subnet = "my-gke-proxy-subnet"
diff --git a/terraform/vpc.tf b/terraform/vpc.tf
@@ -1,13 +1,44 @@
-# VPC
+# https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/using_gke_with_terraform
+# https://cloud.google.com/kubernetes-engine/docs/how-to/deploying-gateways
+
+# Important:
+# The Gateway API is supported on VPC-native clusters only.
+# I needs a proxy-only subnet
+
+# VPC main for GLE Cluster
 resource "google_compute_network" "vpc" {
   name                    = "${var.project_id}-vpc"
-  auto_create_subnetworks = "false"
+  auto_create_subnetworks = false
 }
 
-# Subnet
+# Subnet config for VPC-native in GKE Cluster
 resource "google_compute_subnetwork" "subnet" {
   name          = "${var.project_id}-subnet"
   region        = var.region
+  ip_cidr_range = "10.2.0.0/16"
+  network       = google_compute_network.vpc.id
+  secondary_ip_range {
+    range_name    = "services-range"
+    ip_cidr_range = "192.168.1.0/24"
+  }
+
+  secondary_ip_range {
+    range_name    = "pod-ranges"
+    ip_cidr_range = "192.168.64.0/22"
+  }
+}
+
+# Proxy-Only Subnet
+resource "google_compute_subnetwork" "managed_proxy_subnet" {
+  role          = "ACTIVE"
+  purpose       = "REGIONAL_MANAGED_PROXY"
+  name          = var.managed_proxy_subnet
+  region        = var.region
   network       = google_compute_network.vpc.name
-  ip_cidr_range = "10.10.0.0/24"
+  ip_cidr_range = "10.0.32.0/20"
+
+  depends_on = [google_compute_network.vpc]
 }
+
+# Verify your proxy-only subnet
+# gcloud compute networks subnets describe my-gke-proxy-subnet --region=us-central1