Add cloud build configuration to build workers

containers/buildkite-premerge-debian/.dockerignore

@@ -0,0 +1 @@
+cloudbuild.yaml

containers/buildkite-premerge-debian/cloudbuild.yaml

@@ -0,0 +1,6 @@
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['build', '-t', 'gcr.io/${PROJECT_ID}/buildkite-premerge-debian:latest', '.']
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args: ['push', 'gcr.io/${PROJECT_ID}/buildkite-premerge-debian:latest']

containers/buildkite-premerge-windows/.dockerignore

@@ -0,0 +1 @@
+cloudbuild.yaml

containers/buildkite-premerge-windows/Dockerfile

@@ -93,7 +93,7 @@ RUN powershell -Command `
 RUN git config --system core.longpaths true & `
     git config --global core.autocrlf false
 
-# For debugging of files beeing locked by some processes.
+# handle for debugging of files beeing locked by some processes.
 RUN choco install -y handle
 COPY start_agent.ps1 C:\scripts\
 COPY pre-checkout.bat c:\buildkite-agent\hooks\

containers/buildkite-premerge-windows/cloudbuild.yaml

@@ -0,0 +1,16 @@
+#TODO replace network, subnetwork, region, zone and image with the placeholders. Trigger this from the terraform ?
+#windows-server-2019-dc-core-for-containers-v20230113
+#windows-server-2019-dc-for-containers-v20230113
+#TODO get latest image in family
+#TODO check different machine types and timings
+steps:
+  - name: 'gcr.io/${PROJECT_ID}/windows-builder'
+    args: ['--image','windows-cloud/global/images/windows-server-2019-dc-core-for-containers-v20230113',
+          '--network', 'vpc-network',
+          '--subnetwork', 'subnetwork',
+          '--region', 'europe-west3',
+          '--zone', 'europe-west3-c',
+          '--machineType', "n2-standard-16",
+          '--diskType', 'pd-ssd',
+          '--command', 'gcloud auth configure-docker --quiet && docker build -t gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest . && docker push gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest']
+timeout: 7200s

terraform/README.md

@@ -1,35 +1,53 @@
-ToDo:
+#ToDo:
 Move secrets to gcp secrets
-Add readme
-[]Add dependencies for kubernetes https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775
+Format readme in md
 Cloud build for terraform
-Check billing alerts
-Push images to the cluster or to central location
 [?]Wait for kubernetes plugin to finish apply
+Add readme if the budget is exceeded
 
-1st run (bootstrap)
+Be aware, the actions you execute on your gcp project will generate some cost.
 
-Copy variables.tfvars from variables.tfvars_example
-Insert project-id and billing-account
-Insert secret values in the variables.tfvars file or insert values on runtime when using terraform apply
+#Permissions
+TODO
+
+#1st run (bootstrap)
+
+Copy `variables.tfvars` from `variables.tfvars_example`
+Replace the placeholders for `project-id` and `billing-account` in `variables.tfvars`
+Insert secret values in the `variables.tfvars` file or insert values on runtime when using terraform plan or apply
 
 Initialise terraform
-Comment out everything in backend.tf file to use local state
-terraform init
+Comment out everything in `backend.tf` file to use local state for the first run as the bucket for storing the state is not created.
+```terraform init```
 
 Create the state bucket
-terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"
-To disable the conformation use --auto-aprove flag
+```terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"```
+To disable the conformation use `--auto-aprove` flag
 
-Move the state to the bucket
-Uncomment everything in backend.tf file to use remote state
-export PROJECT_ID="<PROJECT_ID>"
-terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"
+##Move the state to the bucket.
+Uncomment everything in `backend.tf` file to use remote state with newly created bucket.
+```export PROJECT_ID="<PROJECT_ID>"```
+```terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"```
 
-Create the cluster. Due to the problem described here [https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775] kubernetes provider requires cluster to be created. So we have to do the apply in 2 runs using the -target flag
-terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"
+Create the cluster. Due to the problem described [here](https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775) terraform kubernetes provider requires kubernetes cluster to be created first. So to create the cluster without applying kubernetes resources we will do the apply in 2 runs using the `-target` flag.
+```terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"```
 
-To deploy build slaves you need to have slaves docker images in your project. (TODO or we'll move them to the central project)
+##Creating worker images
+To deploy build workers you need the worker docker images in your project.
+TODO cloud build SA permissions
 
-Create the rest of the resources
-terraform apply -var-file="variables.tfvars"
+###Linux worker image
+Execute cloud build to build Linux worker:
+```gcloud builds submit --config=containers/buildkite-premerge-debian/cloudbuild.yaml containers/buildkite-premerge-debian/ --project=${PROJECT_ID}```
+
+###Windows worker image
+Build windows cloud builder. Follow the steps described here: [link](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/windows-builder)
+
+Execute cloud build to build Windows worker:
+```gcloud builds submit --config=containers/buildkite-premerge-windows/cloudbuild.yaml containers/buildkite-premerge-windows/ --project=${PROJECT_ID}```
+
+##Create the rest of the gcp resources including workers in kubernetes pods
+```terraform apply -var-file="variables.tfvars"```
+
+#Budget
+TODO

terraform/kubernetes/linux-agents.yaml

@@ -34,7 +34,7 @@ spec:
     spec:
       containers:
         - name: buildkite-premerge-debian
-          image: gcr.io/${project-id}/buildkite-premerge-debian
+          image: gcr.io/${project-id}/buildkite-premerge-debian:latest
           resources:
             limits:
               cpu: ${cpu-request}

terraform/kubernetes/windows-agents.yaml

@@ -34,7 +34,7 @@ spec:
     spec:
       containers:
         - name: buildkite-premerge-windows
-          image: gcr.io/${project-id}/buildkite-premerge-windows
+          image: gcr.io/${project-id}/buildkite-premerge-windows:latest
           resources:
             limits:
               cpu: ${cpu-request}

terraform/main.tf

@@ -1,5 +1,36 @@
 #todo automatically rebuild buildkite images
 
+data "google_project" "current_project" {
+  project_id = var.project-id
+}
+
+locals {
+  cloud_build_sa_roles = ["roles/storage.objectAdmin", "roles/compute.instanceAdmin", "roles/compute.securityAdmin"]
+}
+
+# data "google_iam_policy" "cloud_build_sa" {
+#   binding {
+#     role = "roles/iam.serviceAccountUser"
+
+#     members = [
+#       "serviceAccount:${data.google_project.current_project.number}-compute@developer.gserviceaccount.com",
+#     ]
+#   }
+# }
+
+# resource "google_service_account_iam_policy" "admin-account-iam" {
+#   service_account_id = "${data.google_project.current_project.id}/serviceAccounts/${data.google_project.current_project.number}@cloudbuild.gserviceaccount.com"
+#   policy_data        = data.google_iam_policy.cloud_build_sa.policy_data
+# }
+
+resource "google_project_iam_member" "cloudbuild_sa_roles" {
+  project = var.project-id
+  for_each = toset(local.cloud_build_sa_roles)
+  role    = each.value
+
+  member = "serviceAccount:${data.google_project.current_project.number}@cloudbuild.gserviceaccount.com"
+}
+
 resource "google_project_service" "cloudbuild_api" {
   service = "cloudbuild.googleapis.com"
 }

terraform/variables.tfvars_example

@@ -3,17 +3,17 @@ billing-account              = ""
 billing-budget               = 25000
 billing-admins               = {"test": "test@test.com"}
 
-linux-agents-machine-type    = "e2-standard-8"
-linux-agents-count           = 1
-linux-agents-build-queue     = "linux-test"
-linux-agents-cpu-request     = "6"
-linux-agents-mem-request     = "8Gi"
+#linux-agents-machine-type    = "e2-standard-8"
+#linux-agents-count           = 1
+#linux-agents-build-queue     = "linux-test"
+#linux-agents-cpu-request     = "6"
+#linux-agents-mem-request     = "8Gi"
 
-windows-agents-machine-type    = "e2-standard-8"
-windows-agents-count           = 1
-windows-agents-build-queue     = "windows-test"
-windows-agents-cpu-request     = "6"
-windows-agents-mem-request     = "8Gi"
+#windows-agents-machine-type    = "e2-standard-8"
+#windows-agents-count           = 1
+#windows-agents-build-queue     = "windows-test"
+#windows-agents-cpu-request     = "6"
+#windows-agents-mem-request     = "8Gi"
 
 buildkite-api-token-readonly = ""
 buildkite-agent-token        = ""