Bugfixing + Enabled billing alerts in TF

terraform/README.md

@@ -1,4 +1,35 @@
-export PROJECT_ID="pre-merge-checks"
+ToDo:
+Move secrets to gcp secrets
+Add readme
+[]Add dependencies for kubernetes https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775
+Cloud build for terraform
+Check billing alerts
+Push images to the cluster or to central location
+[?]Wait for kubernetes plugin to finish apply
+
+1st run (bootstrap)
+
+Copy variables.tfvars from variables.tfvars_example
+Insert project-id and billing-account
+Insert secret values in the variables.tfvars file or insert values on runtime when using terraform apply
+
+Initialise terraform
+Comment out everything in backend.tf file to use local state
+terraform init
+
+Create the state bucket
+terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"
+To disable the conformation use --auto-aprove flag
+
+Move the state to the bucket
+Uncomment everything in backend.tf file to use remote state
+export PROJECT_ID="<PROJECT_ID>"
 terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"
 
-terraform apply -var-file="variables.tfvars" --auto-approve
+Create the cluster. Due to the problem described here [https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775] kubernetes provider requires cluster to be created. So we have to do the apply in 2 runs using the -target flag
+terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"
+
+To deploy build slaves you need to have slaves docker images in your project. (TODO or we'll move them to the central project)
+
+Create the rest of the resources
+terraform apply -var-file="variables.tfvars"

terraform/billing.tf

@@ -1,47 +1,48 @@
-# #todo fix billing alert creation
-# data "google_billing_account" "account" {
-#   billing_account = "01E34D-BF37C6-8137F6"
-# }
+#todo fix billing alert creation
+data "google_billing_account" "account" {
+  billing_account = var.billing-account
+}
 
-# resource "google_billing_budget" "budget" {
-#   billing_account = data.google_billing_account.account.id
-#   display_name = "budget"
-#   amount {
-#     specified_amount {
-#       currency_code = "USD"
-#       units = "25000"
-#     }
-#   }
+resource "google_billing_budget" "budget" {
+  billing_account = data.google_billing_account.account.id
+  display_name    = "budget"
+  amount {
+    specified_amount {
+      currency_code = "USD"
+      units         = var.billing-budget
+    }
+  }
 
-#   budget_filter {
-#     #projects = ["projects/${data.google_project.project.number}"]
-#     credit_types_treatment = "EXCLUDE_ALL_CREDITS"
-#     #services = ["services/24E6-581D-38E5"] # Bigquery
-#   }
+  budget_filter {
+    projects               = ["projects/${var.project-id}"]
+    credit_types_treatment = "EXCLUDE_ALL_CREDITS"
+    #services = ["services/24E6-581D-38E5"] # Bigquery
+  }
 
-#   threshold_rules {
-#     threshold_percent = 0.5
-#   }
-#   threshold_rules {
-#     threshold_percent = 0.9
-#   }
-#   threshold_rules {
-#     threshold_percent = 1.0
-#   }
+  threshold_rules {
+    threshold_percent = 0.5
+  }
+  threshold_rules {
+    threshold_percent = 0.9
+  }
+  threshold_rules {
+    threshold_percent = 1.0
+  }
 
-#   # all_updates_rule {
-#   #   monitoring_notification_channels = [
-#   #     google_monitoring_notification_channel.notification_channel.id,
-#   #   ]
-#   #   disable_default_iam_recipients = true
-#   # }
-# }
+  all_updates_rule {
+    monitoring_notification_channels = [
+      for k, v in google_monitoring_notification_channel.notification_channel : google_monitoring_notification_channel.notification_channel[k].id
+    ]
+    disable_default_iam_recipients = true
+  }
+}
 
-# resource "google_monitoring_notification_channel" "notification_channel" {
-#   display_name = "Example Notification Channel"
-#   type         = "email"
+resource "google_monitoring_notification_channel" "notification_channel" {
+  for_each     = var.billing-admins
+  display_name = each.key
+  type         = "email"
 
-#   labels = {
-#     email_address = "address@example.com"
-#   }
-# }
+  labels = {
+    email_address = each.value
+  }
+}

terraform/cluster.tf

@@ -3,6 +3,33 @@ resource "google_service_account" "llvm_premerge_checks_sa" {
   display_name = "Service Account used with the gke cluster"
 }
 
+resource "google_project_iam_binding" "sa_gcr_reader_role" {
+  project = var.project-id
+  role    = "roles/storage.objectViewer"
+
+  members = [
+    "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
+  ]
+}
+
+resource "google_project_iam_binding" "sa_mertics_writer_role" {
+  project = var.project-id
+  role    = "roles/monitoring.metricWriter"
+
+  members = [
+    "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
+  ]
+}
+
+resource "google_project_iam_binding" "sa_logging_writer_role" {
+  project = var.project-id
+  role    = "roles/logging.logWriter"
+
+  members = [
+    "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
+  ]
+}
+
 resource "google_container_cluster" "llvm_premerge_checks_cluster" {
   name = "llvm-premerge-checks-cluster"
 
@@ -31,6 +58,7 @@ resource "google_container_cluster" "llvm_premerge_checks_cluster" {
     cluster_secondary_range_name  = "pods"
     services_secondary_range_name = "services"
   }
+  depends_on = [google_project_service.compute_api, google_project_service.container_api]
 }
 
 resource "google_container_node_pool" "linux_agents_nodepool" {
@@ -40,9 +68,11 @@ resource "google_container_node_pool" "linux_agents_nodepool" {
   node_config {
     machine_type = var.linux-agents-machine-type
     image_type   = "cos_containerd"
+    disk_size_gb = 500
+    disk_type    = "pd-ssd"
 
     #todo: assign right permissions and use custom service account
-    service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email
+    service_account = google_service_account.llvm_premerge_checks_sa.email
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
@@ -61,9 +91,11 @@ resource "google_container_node_pool" "windows_agents_nodepool" {
   node_config {
     machine_type = var.windows-agents-machine-type
     image_type   = "windows_ltsc_containerd" # todo ltsc or sac ?
+    disk_size_gb = 500
+    disk_type    = "pd-ssd"
 
     #todo: assign right permissions and use custom service account
-    service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email
+    service_account = google_service_account.llvm_premerge_checks_sa.email
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
@@ -113,9 +145,19 @@ resource "kubernetes_manifest" "buildkite_conduit_api_token_secret" {
 resource "kubernetes_manifest" "buildkite_linux_agent" {
   manifest   = yamldecode(templatefile("kubernetes/linux-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.linux_agents_nodepool.name, build-queue = var.linux-agents-build-queue, cpu-request = var.linux-agents-cpu-request, mem-request = var.linux-agents-mem-request, replicas-count = var.linux-agents-count }))
   depends_on = [kubernetes_manifest.buildkite_namespace]
+  # wait {
+  #   fields = {
+  #     "status.phase" = "Running"
+  #   }
+  # }
 }
 
 resource "kubernetes_manifest" "buildkite_windows_agent" {
   manifest   = yamldecode(templatefile("kubernetes/windows-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.windows_agents_nodepool.name, build-queue = var.windows-agents-build-queue, cpu-request = var.windows-agents-cpu-request, mem-request = var.windows-agents-mem-request, replicas-count = var.windows-agents-count }))
   depends_on = [kubernetes_manifest.buildkite_namespace]
+  # wait {
+  #   fields = {
+  #     "status.phase" = "Running"
+  #   }
+  # }
 }

terraform/kubernetes/linux-agents.yaml

@@ -58,7 +58,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=${build-queue},name=$(POD_NAME)"
+              value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}"
             - name: BUILDKITE_BUILD_PATH
               value: "/var/lib/buildkite-agent/builds"
             - name: CONDUIT_TOKEN

terraform/kubernetes/windows-agents.yaml

@@ -56,7 +56,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=${build-queue},name=$(POD_NAME)"
+              value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}"
             # - name: BUILDKITE_BUILD_PATH
             #   value: "C:\\ws"
             - name: CONDUIT_TOKEN

terraform/main.tf

@@ -4,8 +4,29 @@ resource "google_project_service" "cloudbuild_api" {
   service = "cloudbuild.googleapis.com"
 }
 
+resource "google_project_service" "compute_api" {
+  service = "compute.googleapis.com"
+}
+
+resource "google_project_service" "container_api" {
+  service = "container.googleapis.com"
+}
+
+resource "google_project_service" "cloudresourcemanager_api" {
+  service = "cloudresourcemanager.googleapis.com"
+}
+
+resource "google_project_service" "cloudbilling_api" {
+  service = "cloudbilling.googleapis.com"
+}
+
+resource "google_project_service" "billingbudgets_api" {
+  service = "billingbudgets.googleapis.com"
+}
+
 resource "google_storage_bucket" "terraform_state" {
   name                        = "terraform-state-${var.project-id}"
+  uniform_bucket_level_access = true
   location                    = "EU"
 }
 

terraform/provider.tf

@@ -2,4 +2,6 @@ provider "google" {
   project               = var.project-id
   region                = var.region
   zone                  = var.zone
+  billing_project       = var.project-id
+  user_project_override = true
 }

terraform/variables.tf

@@ -2,6 +2,18 @@ variable "project-id" {
   type = string
 }
 
+variable "billing-account" {
+  type = string
+}
+
+variable "billing-budget" {
+  type = number
+}
+
+variable "billing-admins" {
+  type = map(any)
+}
+
 variable "region" {
   type    = string
   default = "europe-west3"

terraform/variables.tfvars_example

@@ -0,0 +1,23 @@
+project-id                   = ""
+billing-account              = ""
+billing-budget               = 25000
+billing-admins               = {"test": "test@test.com"}
+
+linux-agents-machine-type    = "e2-standard-8"
+linux-agents-count           = 1
+linux-agents-build-queue     = "linux-test"
+linux-agents-cpu-request     = "6"
+linux-agents-mem-request     = "8Gi"
+
+windows-agents-machine-type    = "e2-standard-8"
+windows-agents-count           = 1
+windows-agents-build-queue     = "windows-test"
+windows-agents-cpu-request     = "6"
+windows-agents-mem-request     = "8Gi"
+
+buildkite-api-token-readonly = ""
+buildkite-agent-token        = ""
+conduit-api-token            = ""
+git-id-rsa                   = ""
+id-rsa-pub                   = ""
+git-known-hosts              = ""