From cb2ba571cf12b7d1d4ee3f01fdb6b7c820c961f3 Mon Sep 17 00:00:00 2001
From: Grigory <movsic@gmail.com>
Date: Mon, 30 Jan 2023 22:54:53 +0100
Subject: [PATCH] Bugfixing + Enabled billing alerts in TF

---
 terraform/README.md                      | 35 +++++++++-
 terraform/billing.tf                     | 83 ++++++++++++------------
 terraform/cluster.tf                     | 50 ++++++++++++--
 terraform/kubernetes/linux-agents.yaml   |  2 +-
 terraform/kubernetes/windows-agents.yaml |  2 +-
 terraform/main.tf                        | 25 ++++++-
 terraform/provider.tf                    |  8 ++-
 terraform/variables.tf                   | 12 ++++
 terraform/variables.tfvars_example       | 23 +++++++
 9 files changed, 186 insertions(+), 54 deletions(-)
 create mode 100644 terraform/variables.tfvars_example

diff --git a/terraform/README.md b/terraform/README.md
index 6326c66..efb10d2 100644
--- a/terraform/README.md
+++ b/terraform/README.md
@@ -1,4 +1,35 @@
-export PROJECT_ID="pre-merge-checks"
+ToDo:
+Move secrets to gcp secrets
+Add readme
+[]Add dependencies for kubernetes https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775
+Cloud build for terraform
+Check billing alerts
+Push images to the cluster or to central location
+[?]Wait for kubernetes plugin to finish apply
+
+1st run (bootstrap)
+
+Copy variables.tfvars from variables.tfvars_example
+Insert project-id and billing-account
+Insert secret values in the variables.tfvars file or insert values on runtime when using terraform apply
+
+Initialise terraform
+Comment out everything in backend.tf file to use local state
+terraform init
+
+Create the state bucket
+terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"
+To disable the conformation use --auto-aprove flag
+
+Move the state to the bucket
+Uncomment everything in backend.tf file to use remote state
+export PROJECT_ID="<PROJECT_ID>"
 terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"
 
-terraform apply -var-file="variables.tfvars" --auto-approve
\ No newline at end of file
+Create the cluster. Due to the problem described here [https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775] kubernetes provider requires cluster to be created. So we have to do the apply in 2 runs using the -target flag
+terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"
+
+To deploy build slaves you need to have slaves docker images in your project. (TODO or we'll move them to the central project)
+
+Create the rest of the resources
+terraform apply -var-file="variables.tfvars"
\ No newline at end of file
diff --git a/terraform/billing.tf b/terraform/billing.tf
index 007e566..a7c39c2 100644
--- a/terraform/billing.tf
+++ b/terraform/billing.tf
@@ -1,47 +1,48 @@
-# #todo fix billing alert creation
-# data "google_billing_account" "account" {
-#   billing_account = "01E34D-BF37C6-8137F6"
-# }
+#todo fix billing alert creation
+data "google_billing_account" "account" {
+  billing_account = var.billing-account
+}
 
-# resource "google_billing_budget" "budget" {
-#   billing_account = data.google_billing_account.account.id
-#   display_name = "budget"
-#   amount {
-#     specified_amount {
-#       currency_code = "USD"
-#       units = "25000"
-#     }
-#   }
+resource "google_billing_budget" "budget" {
+  billing_account = data.google_billing_account.account.id
+  display_name    = "budget"
+  amount {
+    specified_amount {
+      currency_code = "USD"
+      units         = var.billing-budget
+    }
+  }
 
-#   budget_filter {
-#     #projects = ["projects/${data.google_project.project.number}"]
-#     credit_types_treatment = "EXCLUDE_ALL_CREDITS"
-#     #services = ["services/24E6-581D-38E5"] # Bigquery
-#   }
+  budget_filter {
+    projects               = ["projects/${var.project-id}"]
+    credit_types_treatment = "EXCLUDE_ALL_CREDITS"
+    #services = ["services/24E6-581D-38E5"] # Bigquery
+  }
 
-#   threshold_rules {
-#     threshold_percent = 0.5
-#   }
-#   threshold_rules {
-#     threshold_percent = 0.9
-#   }
-#   threshold_rules {
-#     threshold_percent = 1.0
-#   }
+  threshold_rules {
+    threshold_percent = 0.5
+  }
+  threshold_rules {
+    threshold_percent = 0.9
+  }
+  threshold_rules {
+    threshold_percent = 1.0
+  }
 
-#   # all_updates_rule {
-#   #   monitoring_notification_channels = [
-#   #     google_monitoring_notification_channel.notification_channel.id,
-#   #   ]
-#   #   disable_default_iam_recipients = true
-#   # }
-# }
+  all_updates_rule {
+    monitoring_notification_channels = [
+      for k, v in google_monitoring_notification_channel.notification_channel : google_monitoring_notification_channel.notification_channel[k].id
+    ]
+    disable_default_iam_recipients = true
+  }
+}
 
-# resource "google_monitoring_notification_channel" "notification_channel" {
-#   display_name = "Example Notification Channel"
-#   type         = "email"
+resource "google_monitoring_notification_channel" "notification_channel" {
+  for_each     = var.billing-admins
+  display_name = each.key
+  type         = "email"
 
-#   labels = {
-#     email_address = "address@example.com"
-#   }
-# }
\ No newline at end of file
+  labels = {
+    email_address = each.value
+  }
+}
\ No newline at end of file
diff --git a/terraform/cluster.tf b/terraform/cluster.tf
index a0e34a1..74c160a 100644
--- a/terraform/cluster.tf
+++ b/terraform/cluster.tf
@@ -3,6 +3,33 @@ resource "google_service_account" "llvm_premerge_checks_sa" {
   display_name = "Service Account used with the gke cluster"
 }
 
+resource "google_project_iam_binding" "sa_gcr_reader_role" {
+  project = var.project-id
+  role    = "roles/storage.objectViewer"
+
+  members = [
+    "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
+  ]
+}
+
+resource "google_project_iam_binding" "sa_mertics_writer_role" {
+  project = var.project-id
+  role    = "roles/monitoring.metricWriter"
+
+  members = [
+    "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
+  ]
+}
+
+resource "google_project_iam_binding" "sa_logging_writer_role" {
+  project = var.project-id
+  role    = "roles/logging.logWriter"
+
+  members = [
+    "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
+  ]
+}
+
 resource "google_container_cluster" "llvm_premerge_checks_cluster" {
   name = "llvm-premerge-checks-cluster"
 
@@ -31,6 +58,7 @@ resource "google_container_cluster" "llvm_premerge_checks_cluster" {
     cluster_secondary_range_name  = "pods"
     services_secondary_range_name = "services"
   }
+  depends_on = [google_project_service.compute_api, google_project_service.container_api]
 }
 
 resource "google_container_node_pool" "linux_agents_nodepool" {
@@ -39,10 +67,12 @@ resource "google_container_node_pool" "linux_agents_nodepool" {
 
   node_config {
     machine_type = var.linux-agents-machine-type
-    image_type = "cos_containerd"
+    image_type   = "cos_containerd"
+    disk_size_gb = 500
+    disk_type    = "pd-ssd"
 
     #todo: assign right permissions and use custom service account
-    service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email
+    service_account = google_service_account.llvm_premerge_checks_sa.email
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
@@ -60,10 +90,12 @@ resource "google_container_node_pool" "windows_agents_nodepool" {
 
   node_config {
     machine_type = var.windows-agents-machine-type
-    image_type = "windows_ltsc_containerd" # todo ltsc or sac ?
+    image_type   = "windows_ltsc_containerd" # todo ltsc or sac ?
+    disk_size_gb = 500
+    disk_type    = "pd-ssd"
 
     #todo: assign right permissions and use custom service account
-    service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email
+    service_account = google_service_account.llvm_premerge_checks_sa.email
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
@@ -113,9 +145,19 @@ resource "kubernetes_manifest" "buildkite_conduit_api_token_secret" {
 resource "kubernetes_manifest" "buildkite_linux_agent" {
   manifest   = yamldecode(templatefile("kubernetes/linux-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.linux_agents_nodepool.name, build-queue = var.linux-agents-build-queue, cpu-request = var.linux-agents-cpu-request, mem-request = var.linux-agents-mem-request, replicas-count = var.linux-agents-count }))
   depends_on = [kubernetes_manifest.buildkite_namespace]
+  # wait {
+  #   fields = {
+  #     "status.phase" = "Running"
+  #   }
+  # }
 }
 
 resource "kubernetes_manifest" "buildkite_windows_agent" {
   manifest   = yamldecode(templatefile("kubernetes/windows-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.windows_agents_nodepool.name, build-queue = var.windows-agents-build-queue, cpu-request = var.windows-agents-cpu-request, mem-request = var.windows-agents-mem-request, replicas-count = var.windows-agents-count }))
   depends_on = [kubernetes_manifest.buildkite_namespace]
+  # wait {
+  #   fields = {
+  #     "status.phase" = "Running"
+  #   }
+  # }
 }
\ No newline at end of file
diff --git a/terraform/kubernetes/linux-agents.yaml b/terraform/kubernetes/linux-agents.yaml
index 55d9c16..a3b37b6 100644
--- a/terraform/kubernetes/linux-agents.yaml
+++ b/terraform/kubernetes/linux-agents.yaml
@@ -58,7 +58,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=${build-queue},name=$(POD_NAME)"
+              value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}"
             - name: BUILDKITE_BUILD_PATH
               value: "/var/lib/buildkite-agent/builds"
             - name: CONDUIT_TOKEN
diff --git a/terraform/kubernetes/windows-agents.yaml b/terraform/kubernetes/windows-agents.yaml
index 252d69b..740cdc7 100644
--- a/terraform/kubernetes/windows-agents.yaml
+++ b/terraform/kubernetes/windows-agents.yaml
@@ -56,7 +56,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=${build-queue},name=$(POD_NAME)"
+              value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}"
             # - name: BUILDKITE_BUILD_PATH
             #   value: "C:\\ws"
             - name: CONDUIT_TOKEN
diff --git a/terraform/main.tf b/terraform/main.tf
index cfed147..314f35b 100644
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -4,9 +4,30 @@ resource "google_project_service" "cloudbuild_api" {
   service = "cloudbuild.googleapis.com"
 }
 
+resource "google_project_service" "compute_api" {
+  service = "compute.googleapis.com"
+}
+
+resource "google_project_service" "container_api" {
+  service = "container.googleapis.com"
+}
+
+resource "google_project_service" "cloudresourcemanager_api" {
+  service = "cloudresourcemanager.googleapis.com"
+}
+
+resource "google_project_service" "cloudbilling_api" {
+  service = "cloudbilling.googleapis.com"
+}
+
+resource "google_project_service" "billingbudgets_api" {
+  service = "billingbudgets.googleapis.com"
+}
+
 resource "google_storage_bucket" "terraform_state" {
-  name     = "terraform-state-${var.project-id}"
-  location = "EU"
+  name                        = "terraform-state-${var.project-id}"
+  uniform_bucket_level_access = true
+  location                    = "EU"
 }
 
 resource "google_compute_network" "vpc_network" {
diff --git a/terraform/provider.tf b/terraform/provider.tf
index ad8daba..1a55510 100644
--- a/terraform/provider.tf
+++ b/terraform/provider.tf
@@ -1,5 +1,7 @@
 provider "google" {
-  project = var.project-id
-  region  = var.region
-  zone    = var.zone
+  project               = var.project-id
+  region                = var.region
+  zone                  = var.zone
+  billing_project       = var.project-id
+  user_project_override = true
 }
\ No newline at end of file
diff --git a/terraform/variables.tf b/terraform/variables.tf
index 86864d5..46c1ae9 100644
--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@@ -2,6 +2,18 @@ variable "project-id" {
   type = string
 }
 
+variable "billing-account" {
+  type = string
+}
+
+variable "billing-budget" {
+  type = number
+}
+
+variable "billing-admins" {
+  type = map(any)
+}
+
 variable "region" {
   type    = string
   default = "europe-west3"
diff --git a/terraform/variables.tfvars_example b/terraform/variables.tfvars_example
new file mode 100644
index 0000000..d831fdd
--- /dev/null
+++ b/terraform/variables.tfvars_example
@@ -0,0 +1,23 @@
+project-id                   = ""
+billing-account              = ""
+billing-budget               = 25000
+billing-admins               = {"test": "test@test.com"}
+
+linux-agents-machine-type    = "e2-standard-8"
+linux-agents-count           = 1
+linux-agents-build-queue     = "linux-test"
+linux-agents-cpu-request     = "6"
+linux-agents-mem-request     = "8Gi"
+
+windows-agents-machine-type    = "e2-standard-8"
+windows-agents-count           = 1
+windows-agents-build-queue     = "windows-test"
+windows-agents-cpu-request     = "6"
+windows-agents-mem-request     = "8Gi"
+
+buildkite-api-token-readonly = ""
+buildkite-agent-token        = ""
+conduit-api-token            = ""
+git-id-rsa                   = ""
+id-rsa-pub                   = ""
+git-known-hosts              = ""
\ No newline at end of file