Bugfixing + Enabled billing alerts in TF
This commit is contained in:
Grigory
Mikhail Goncharov
parent
47116b9841
commit
cb2ba571cf
9 changed files with 186 additions and 54 deletions
|
|
@ -1,4 +1,35 @@
|
|||
export PROJECT_ID="pre-merge-checks"
|
||||
ToDo:
|
||||
Move secrets to gcp secrets
|
||||
Add readme
|
||||
[]Add dependencies for kubernetes https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775
|
||||
Cloud build for terraform
|
||||
Check billing alerts
|
||||
Push images to the cluster or to central location
|
||||
[?]Wait for kubernetes plugin to finish apply
|
||||
|
||||
1st run (bootstrap)
|
||||
|
||||
Copy variables.tfvars from variables.tfvars_example
|
||||
Insert project-id and billing-account
|
||||
Insert secret values in the variables.tfvars file or insert values on runtime when using terraform apply
|
||||
|
||||
Initialise terraform
|
||||
Comment out everything in backend.tf file to use local state
|
||||
terraform init
|
||||
|
||||
Create the state bucket
|
||||
terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"
|
||||
To disable the conformation use --auto-aprove flag
|
||||
|
||||
Move the state to the bucket
|
||||
Uncomment everything in backend.tf file to use remote state
|
||||
export PROJECT_ID="<PROJECT_ID>"
|
||||
terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"
|
||||
|
||||
terraform apply -var-file="variables.tfvars" --auto-approve
|
||||
Create the cluster. Due to the problem described here [https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775] kubernetes provider requires cluster to be created. So we have to do the apply in 2 runs using the -target flag
|
||||
terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"
|
||||
|
||||
To deploy build slaves you need to have slaves docker images in your project. (TODO or we'll move them to the central project)
|
||||
|
||||
Create the rest of the resources
|
||||
terraform apply -var-file="variables.tfvars"
|
||||
|
|
@ -1,47 +1,48 @@
|
|||
# #todo fix billing alert creation
|
||||
# data "google_billing_account" "account" {
|
||||
# billing_account = "01E34D-BF37C6-8137F6"
|
||||
# }
|
||||
#todo fix billing alert creation
|
||||
data "google_billing_account" "account" {
|
||||
billing_account = var.billing-account
|
||||
}
|
||||
|
||||
# resource "google_billing_budget" "budget" {
|
||||
# billing_account = data.google_billing_account.account.id
|
||||
# display_name = "budget"
|
||||
# amount {
|
||||
# specified_amount {
|
||||
# currency_code = "USD"
|
||||
# units = "25000"
|
||||
# }
|
||||
# }
|
||||
resource "google_billing_budget" "budget" {
|
||||
billing_account = data.google_billing_account.account.id
|
||||
display_name = "budget"
|
||||
amount {
|
||||
specified_amount {
|
||||
currency_code = "USD"
|
||||
units = var.billing-budget
|
||||
}
|
||||
}
|
||||
|
||||
# budget_filter {
|
||||
# #projects = ["projects/${data.google_project.project.number}"]
|
||||
# credit_types_treatment = "EXCLUDE_ALL_CREDITS"
|
||||
# #services = ["services/24E6-581D-38E5"] # Bigquery
|
||||
# }
|
||||
budget_filter {
|
||||
projects = ["projects/${var.project-id}"]
|
||||
credit_types_treatment = "EXCLUDE_ALL_CREDITS"
|
||||
#services = ["services/24E6-581D-38E5"] # Bigquery
|
||||
}
|
||||
|
||||
# threshold_rules {
|
||||
# threshold_percent = 0.5
|
||||
# }
|
||||
# threshold_rules {
|
||||
# threshold_percent = 0.9
|
||||
# }
|
||||
# threshold_rules {
|
||||
# threshold_percent = 1.0
|
||||
# }
|
||||
threshold_rules {
|
||||
threshold_percent = 0.5
|
||||
}
|
||||
threshold_rules {
|
||||
threshold_percent = 0.9
|
||||
}
|
||||
threshold_rules {
|
||||
threshold_percent = 1.0
|
||||
}
|
||||
|
||||
# # all_updates_rule {
|
||||
# # monitoring_notification_channels = [
|
||||
# # google_monitoring_notification_channel.notification_channel.id,
|
||||
# # ]
|
||||
# # disable_default_iam_recipients = true
|
||||
# # }
|
||||
# }
|
||||
all_updates_rule {
|
||||
monitoring_notification_channels = [
|
||||
for k, v in google_monitoring_notification_channel.notification_channel : google_monitoring_notification_channel.notification_channel[k].id
|
||||
]
|
||||
disable_default_iam_recipients = true
|
||||
}
|
||||
}
|
||||
|
||||
# resource "google_monitoring_notification_channel" "notification_channel" {
|
||||
# display_name = "Example Notification Channel"
|
||||
# type = "email"
|
||||
resource "google_monitoring_notification_channel" "notification_channel" {
|
||||
for_each = var.billing-admins
|
||||
display_name = each.key
|
||||
type = "email"
|
||||
|
||||
# labels = {
|
||||
# email_address = "address@example.com"
|
||||
# }
|
||||
# }
|
||||
labels = {
|
||||
email_address = each.value
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,33 @@ resource "google_service_account" "llvm_premerge_checks_sa" {
|
|||
display_name = "Service Account used with the gke cluster"
|
||||
}
|
||||
|
||||
resource "google_project_iam_binding" "sa_gcr_reader_role" {
|
||||
project = var.project-id
|
||||
role = "roles/storage.objectViewer"
|
||||
|
||||
members = [
|
||||
"serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
|
||||
]
|
||||
}
|
||||
|
||||
resource "google_project_iam_binding" "sa_mertics_writer_role" {
|
||||
project = var.project-id
|
||||
role = "roles/monitoring.metricWriter"
|
||||
|
||||
members = [
|
||||
"serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
|
||||
]
|
||||
}
|
||||
|
||||
resource "google_project_iam_binding" "sa_logging_writer_role" {
|
||||
project = var.project-id
|
||||
role = "roles/logging.logWriter"
|
||||
|
||||
members = [
|
||||
"serviceAccount:${google_service_account.llvm_premerge_checks_sa.email}"
|
||||
]
|
||||
}
|
||||
|
||||
resource "google_container_cluster" "llvm_premerge_checks_cluster" {
|
||||
name = "llvm-premerge-checks-cluster"
|
||||
|
||||
|
|
@ -31,6 +58,7 @@ resource "google_container_cluster" "llvm_premerge_checks_cluster" {
|
|||
cluster_secondary_range_name = "pods"
|
||||
services_secondary_range_name = "services"
|
||||
}
|
||||
depends_on = [google_project_service.compute_api, google_project_service.container_api]
|
||||
}
|
||||
|
||||
resource "google_container_node_pool" "linux_agents_nodepool" {
|
||||
|
|
@ -39,10 +67,12 @@ resource "google_container_node_pool" "linux_agents_nodepool" {
|
|||
|
||||
node_config {
|
||||
machine_type = var.linux-agents-machine-type
|
||||
image_type = "cos_containerd"
|
||||
image_type = "cos_containerd"
|
||||
disk_size_gb = 500
|
||||
disk_type = "pd-ssd"
|
||||
|
||||
#todo: assign right permissions and use custom service account
|
||||
service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email
|
||||
service_account = google_service_account.llvm_premerge_checks_sa.email
|
||||
oauth_scopes = [
|
||||
"https://www.googleapis.com/auth/cloud-platform"
|
||||
]
|
||||
|
|
@ -60,10 +90,12 @@ resource "google_container_node_pool" "windows_agents_nodepool" {
|
|||
|
||||
node_config {
|
||||
machine_type = var.windows-agents-machine-type
|
||||
image_type = "windows_ltsc_containerd" # todo ltsc or sac ?
|
||||
image_type = "windows_ltsc_containerd" # todo ltsc or sac ?
|
||||
disk_size_gb = 500
|
||||
disk_type = "pd-ssd"
|
||||
|
||||
#todo: assign right permissions and use custom service account
|
||||
service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email
|
||||
service_account = google_service_account.llvm_premerge_checks_sa.email
|
||||
oauth_scopes = [
|
||||
"https://www.googleapis.com/auth/cloud-platform"
|
||||
]
|
||||
|
|
@ -113,9 +145,19 @@ resource "kubernetes_manifest" "buildkite_conduit_api_token_secret" {
|
|||
resource "kubernetes_manifest" "buildkite_linux_agent" {
|
||||
manifest = yamldecode(templatefile("kubernetes/linux-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.linux_agents_nodepool.name, build-queue = var.linux-agents-build-queue, cpu-request = var.linux-agents-cpu-request, mem-request = var.linux-agents-mem-request, replicas-count = var.linux-agents-count }))
|
||||
depends_on = [kubernetes_manifest.buildkite_namespace]
|
||||
# wait {
|
||||
# fields = {
|
||||
# "status.phase" = "Running"
|
||||
# }
|
||||
# }
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "buildkite_windows_agent" {
|
||||
manifest = yamldecode(templatefile("kubernetes/windows-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.windows_agents_nodepool.name, build-queue = var.windows-agents-build-queue, cpu-request = var.windows-agents-cpu-request, mem-request = var.windows-agents-mem-request, replicas-count = var.windows-agents-count }))
|
||||
depends_on = [kubernetes_manifest.buildkite_namespace]
|
||||
# wait {
|
||||
# fields = {
|
||||
# "status.phase" = "Running"
|
||||
# }
|
||||
# }
|
||||
}
|
||||
|
|
@ -58,7 +58,7 @@ spec:
|
|||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: BUILDKITE_AGENT_TAGS
|
||||
value: "queue=${build-queue},name=$(POD_NAME)"
|
||||
value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}"
|
||||
- name: BUILDKITE_BUILD_PATH
|
||||
value: "/var/lib/buildkite-agent/builds"
|
||||
- name: CONDUIT_TOKEN
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ spec:
|
|||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: BUILDKITE_AGENT_TAGS
|
||||
value: "queue=${build-queue},name=$(POD_NAME)"
|
||||
value: "queue=${build-queue},name=$(POD_NAME),project=${project-id}"
|
||||
# - name: BUILDKITE_BUILD_PATH
|
||||
# value: "C:\\ws"
|
||||
- name: CONDUIT_TOKEN
|
||||
|
|
|
|||
|
|
@ -4,9 +4,30 @@ resource "google_project_service" "cloudbuild_api" {
|
|||
service = "cloudbuild.googleapis.com"
|
||||
}
|
||||
|
||||
resource "google_project_service" "compute_api" {
|
||||
service = "compute.googleapis.com"
|
||||
}
|
||||
|
||||
resource "google_project_service" "container_api" {
|
||||
service = "container.googleapis.com"
|
||||
}
|
||||
|
||||
resource "google_project_service" "cloudresourcemanager_api" {
|
||||
service = "cloudresourcemanager.googleapis.com"
|
||||
}
|
||||
|
||||
resource "google_project_service" "cloudbilling_api" {
|
||||
service = "cloudbilling.googleapis.com"
|
||||
}
|
||||
|
||||
resource "google_project_service" "billingbudgets_api" {
|
||||
service = "billingbudgets.googleapis.com"
|
||||
}
|
||||
|
||||
resource "google_storage_bucket" "terraform_state" {
|
||||
name = "terraform-state-${var.project-id}"
|
||||
location = "EU"
|
||||
name = "terraform-state-${var.project-id}"
|
||||
uniform_bucket_level_access = true
|
||||
location = "EU"
|
||||
}
|
||||
|
||||
resource "google_compute_network" "vpc_network" {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
provider "google" {
|
||||
project = var.project-id
|
||||
region = var.region
|
||||
zone = var.zone
|
||||
project = var.project-id
|
||||
region = var.region
|
||||
zone = var.zone
|
||||
billing_project = var.project-id
|
||||
user_project_override = true
|
||||
}
|
||||
|
|
@ -2,6 +2,18 @@ variable "project-id" {
|
|||
type = string
|
||||
}
|
||||
|
||||
variable "billing-account" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "billing-budget" {
|
||||
type = number
|
||||
}
|
||||
|
||||
variable "billing-admins" {
|
||||
type = map(any)
|
||||
}
|
||||
|
||||
variable "region" {
|
||||
type = string
|
||||
default = "europe-west3"
|
||||
|
|
|
|||
23
terraform/variables.tfvars_example
Normal file
23
terraform/variables.tfvars_example
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
project-id = ""
|
||||
billing-account = ""
|
||||
billing-budget = 25000
|
||||
billing-admins = {"test": "test@test.com"}
|
||||
|
||||
linux-agents-machine-type = "e2-standard-8"
|
||||
linux-agents-count = 1
|
||||
linux-agents-build-queue = "linux-test"
|
||||
linux-agents-cpu-request = "6"
|
||||
linux-agents-mem-request = "8Gi"
|
||||
|
||||
windows-agents-machine-type = "e2-standard-8"
|
||||
windows-agents-count = 1
|
||||
windows-agents-build-queue = "windows-test"
|
||||
windows-agents-cpu-request = "6"
|
||||
windows-agents-mem-request = "8Gi"
|
||||
|
||||
buildkite-api-token-readonly = ""
|
||||
buildkite-agent-token = ""
|
||||
conduit-api-token = ""
|
||||
git-id-rsa = ""
|
||||
id-rsa-pub = ""
|
||||
git-known-hosts = ""
|
||||
Loading…
Reference in a new issue