diff --git a/terraform/README.md b/terraform/README.md index 6326c66..efb10d2 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,4 +1,35 @@ -export PROJECT_ID="pre-merge-checks" +ToDo: +Move secrets to gcp secrets +Add readme +[]Add dependencies for kubernetes https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775 +Cloud build for terraform +Check billing alerts +Push images to the cluster or to central location +[?]Wait for kubernetes plugin to finish apply + +1st run (bootstrap) + +Copy variables.tfvars from variables.tfvars_example +Insert project-id and billing-account +Insert secret values in the variables.tfvars file or insert values on runtime when using terraform apply + +Initialise terraform +Comment out everything in backend.tf file to use local state +terraform init + +Create the state bucket +terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state" +To disable the conformation use --auto-aprove flag + +Move the state to the bucket +Uncomment everything in backend.tf file to use remote state +export PROJECT_ID="" terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state" -terraform apply -var-file="variables.tfvars" --auto-approve \ No newline at end of file +Create the cluster. Due to the problem described here [https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775] kubernetes provider requires cluster to be created. So we have to do the apply in 2 runs using the -target flag +terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster" + +To deploy build slaves you need to have slaves docker images in your project. (TODO or we'll move them to the central project) + +Create the rest of the resources +terraform apply -var-file="variables.tfvars" \ No newline at end of file diff --git a/terraform/billing.tf b/terraform/billing.tf index 007e566..a7c39c2 100644 --- a/terraform/billing.tf +++ b/terraform/billing.tf @@ -1,47 +1,48 @@ -# #todo fix billing alert creation -# data "google_billing_account" "account" { -# billing_account = "01E34D-BF37C6-8137F6" -# } +#todo fix billing alert creation +data "google_billing_account" "account" { + billing_account = var.billing-account +} -# resource "google_billing_budget" "budget" { -# billing_account = data.google_billing_account.account.id -# display_name = "budget" -# amount { -# specified_amount { -# currency_code = "USD" -# units = "25000" -# } -# } +resource "google_billing_budget" "budget" { + billing_account = data.google_billing_account.account.id + display_name = "budget" + amount { + specified_amount { + currency_code = "USD" + units = var.billing-budget + } + } -# budget_filter { -# #projects = ["projects/${data.google_project.project.number}"] -# credit_types_treatment = "EXCLUDE_ALL_CREDITS" -# #services = ["services/24E6-581D-38E5"] # Bigquery -# } + budget_filter { + projects = ["projects/${var.project-id}"] + credit_types_treatment = "EXCLUDE_ALL_CREDITS" + #services = ["services/24E6-581D-38E5"] # Bigquery + } -# threshold_rules { -# threshold_percent = 0.5 -# } -# threshold_rules { -# threshold_percent = 0.9 -# } -# threshold_rules { -# threshold_percent = 1.0 -# } + threshold_rules { + threshold_percent = 0.5 + } + threshold_rules { + threshold_percent = 0.9 + } + threshold_rules { + threshold_percent = 1.0 + } -# # all_updates_rule { -# # monitoring_notification_channels = [ -# # google_monitoring_notification_channel.notification_channel.id, -# # ] -# # disable_default_iam_recipients = true -# # } -# } + all_updates_rule { + monitoring_notification_channels = [ + for k, v in google_monitoring_notification_channel.notification_channel : google_monitoring_notification_channel.notification_channel[k].id + ] + disable_default_iam_recipients = true + } +} -# resource "google_monitoring_notification_channel" "notification_channel" { -# display_name = "Example Notification Channel" -# type = "email" +resource "google_monitoring_notification_channel" "notification_channel" { + for_each = var.billing-admins + display_name = each.key + type = "email" -# labels = { -# email_address = "address@example.com" -# } -# } \ No newline at end of file + labels = { + email_address = each.value + } +} \ No newline at end of file diff --git a/terraform/cluster.tf b/terraform/cluster.tf index a0e34a1..74c160a 100644 --- a/terraform/cluster.tf +++ b/terraform/cluster.tf @@ -3,6 +3,33 @@ resource "google_service_account" "llvm_premerge_checks_sa" { display_name = "Service Account used with the gke cluster" } +resource "google_project_iam_binding" "sa_gcr_reader_role" { + project = var.project-id + role = "roles/storage.objectViewer" + + members = [ + "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}" + ] +} + +resource "google_project_iam_binding" "sa_mertics_writer_role" { + project = var.project-id + role = "roles/monitoring.metricWriter" + + members = [ + "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}" + ] +} + +resource "google_project_iam_binding" "sa_logging_writer_role" { + project = var.project-id + role = "roles/logging.logWriter" + + members = [ + "serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}" + ] +} + resource "google_container_cluster" "llvm_premerge_checks_cluster" { name = "llvm-premerge-checks-cluster" @@ -31,6 +58,7 @@ resource "google_container_cluster" "llvm_premerge_checks_cluster" { cluster_secondary_range_name = "pods" services_secondary_range_name = "services" } + depends_on = [google_project_service.compute_api, google_project_service.container_api] } resource "google_container_node_pool" "linux_agents_nodepool" { @@ -39,10 +67,12 @@ resource "google_container_node_pool" "linux_agents_nodepool" { node_config { machine_type = var.linux-agents-machine-type - image_type = "cos_containerd" + image_type = "cos_containerd" + disk_size_gb = 500 + disk_type = "pd-ssd" #todo: assign right permissions and use custom service account - service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email + service_account = google_service_account.llvm_premerge_checks_sa.email oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] @@ -60,10 +90,12 @@ resource "google_container_node_pool" "windows_agents_nodepool" { node_config { machine_type = var.windows-agents-machine-type - image_type = "windows_ltsc_containerd" # todo ltsc or sac ? + image_type = "windows_ltsc_containerd" # todo ltsc or sac ? + disk_size_gb = 500 + disk_type = "pd-ssd" #todo: assign right permissions and use custom service account - service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email + service_account = google_service_account.llvm_premerge_checks_sa.email oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] @@ -113,9 +145,19 @@ resource "kubernetes_manifest" "buildkite_conduit_api_token_secret" { resource "kubernetes_manifest" "buildkite_linux_agent" { manifest = yamldecode(templatefile("kubernetes/linux-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.linux_agents_nodepool.name, build-queue = var.linux-agents-build-queue, cpu-request = var.linux-agents-cpu-request, mem-request = var.linux-agents-mem-request, replicas-count = var.linux-agents-count })) depends_on = [kubernetes_manifest.buildkite_namespace] + # wait { + # fields = { + # "status.phase" = "Running" + # } + # } } resource "kubernetes_manifest" "buildkite_windows_agent" { manifest = yamldecode(templatefile("kubernetes/windows-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.windows_agents_nodepool.name, build-queue = var.windows-agents-build-queue, cpu-request = var.windows-agents-cpu-request, mem-request = var.windows-agents-mem-request, replicas-count = var.windows-agents-count })) depends_on = [kubernetes_manifest.buildkite_namespace] + # wait { + # fields = { + # "status.phase" = "Running" + # } + # } } \ No newline at end of file diff --git a/terraform/kubernetes/linux-agents.yaml b/terraform/kubernetes/linux-agents.yaml index 55d9c16..a3b37b6 100644 --- a/terraform/kubernetes/linux-agents.yaml +++ b/terraform/kubernetes/linux-agents.yaml @@ -58,7 +58,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=${build-queue},name=$(POD_NAME)" + value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}" - name: BUILDKITE_BUILD_PATH value: "/var/lib/buildkite-agent/builds" - name: CONDUIT_TOKEN diff --git a/terraform/kubernetes/windows-agents.yaml b/terraform/kubernetes/windows-agents.yaml index 252d69b..740cdc7 100644 --- a/terraform/kubernetes/windows-agents.yaml +++ b/terraform/kubernetes/windows-agents.yaml @@ -56,7 +56,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=${build-queue},name=$(POD_NAME)" + value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}" # - name: BUILDKITE_BUILD_PATH # value: "C:\\ws" - name: CONDUIT_TOKEN diff --git a/terraform/main.tf b/terraform/main.tf index cfed147..314f35b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -4,9 +4,30 @@ resource "google_project_service" "cloudbuild_api" { service = "cloudbuild.googleapis.com" } +resource "google_project_service" "compute_api" { + service = "compute.googleapis.com" +} + +resource "google_project_service" "container_api" { + service = "container.googleapis.com" +} + +resource "google_project_service" "cloudresourcemanager_api" { + service = "cloudresourcemanager.googleapis.com" +} + +resource "google_project_service" "cloudbilling_api" { + service = "cloudbilling.googleapis.com" +} + +resource "google_project_service" "billingbudgets_api" { + service = "billingbudgets.googleapis.com" +} + resource "google_storage_bucket" "terraform_state" { - name = "terraform-state-${var.project-id}" - location = "EU" + name = "terraform-state-${var.project-id}" + uniform_bucket_level_access = true + location = "EU" } resource "google_compute_network" "vpc_network" { diff --git a/terraform/provider.tf b/terraform/provider.tf index ad8daba..1a55510 100644 --- a/terraform/provider.tf +++ b/terraform/provider.tf @@ -1,5 +1,7 @@ provider "google" { - project = var.project-id - region = var.region - zone = var.zone + project = var.project-id + region = var.region + zone = var.zone + billing_project = var.project-id + user_project_override = true } \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index 86864d5..46c1ae9 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -2,6 +2,18 @@ variable "project-id" { type = string } +variable "billing-account" { + type = string +} + +variable "billing-budget" { + type = number +} + +variable "billing-admins" { + type = map(any) +} + variable "region" { type = string default = "europe-west3" diff --git a/terraform/variables.tfvars_example b/terraform/variables.tfvars_example new file mode 100644 index 0000000..d831fdd --- /dev/null +++ b/terraform/variables.tfvars_example @@ -0,0 +1,23 @@ +project-id = "" +billing-account = "" +billing-budget = 25000 +billing-admins = {"test": "test@test.com"} + +linux-agents-machine-type = "e2-standard-8" +linux-agents-count = 1 +linux-agents-build-queue = "linux-test" +linux-agents-cpu-request = "6" +linux-agents-mem-request = "8Gi" + +windows-agents-machine-type = "e2-standard-8" +windows-agents-count = 1 +windows-agents-build-queue = "windows-test" +windows-agents-cpu-request = "6" +windows-agents-mem-request = "8Gi" + +buildkite-api-token-readonly = "" +buildkite-agent-token = "" +conduit-api-token = "" +git-id-rsa = "" +id-rsa-pub = "" +git-known-hosts = "" \ No newline at end of file