Add cloud build configuration to build workers
This commit is contained in:
parent
cb2ba571cf
commit
48db6e55c0
10 changed files with 108 additions and 35 deletions
1
containers/buildkite-premerge-debian/.dockerignore
Normal file
1
containers/buildkite-premerge-debian/.dockerignore
Normal file
|
@ -0,0 +1 @@
|
||||||
|
cloudbuild.yaml
|
6
containers/buildkite-premerge-debian/cloudbuild.yaml
Normal file
6
containers/buildkite-premerge-debian/cloudbuild.yaml
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
steps:
|
||||||
|
- name: 'gcr.io/cloud-builders/docker'
|
||||||
|
args: ['build', '-t', 'gcr.io/${PROJECT_ID}/buildkite-premerge-debian:latest', '.']
|
||||||
|
|
||||||
|
- name: 'gcr.io/cloud-builders/docker'
|
||||||
|
args: ['push', 'gcr.io/${PROJECT_ID}/buildkite-premerge-debian:latest']
|
1
containers/buildkite-premerge-windows/.dockerignore
Normal file
1
containers/buildkite-premerge-windows/.dockerignore
Normal file
|
@ -0,0 +1 @@
|
||||||
|
cloudbuild.yaml
|
|
@ -93,7 +93,7 @@ RUN powershell -Command `
|
||||||
RUN git config --system core.longpaths true & `
|
RUN git config --system core.longpaths true & `
|
||||||
git config --global core.autocrlf false
|
git config --global core.autocrlf false
|
||||||
|
|
||||||
# For debugging of files beeing locked by some processes.
|
# handle for debugging of files beeing locked by some processes.
|
||||||
RUN choco install -y handle
|
RUN choco install -y handle
|
||||||
COPY start_agent.ps1 C:\scripts\
|
COPY start_agent.ps1 C:\scripts\
|
||||||
COPY pre-checkout.bat c:\buildkite-agent\hooks\
|
COPY pre-checkout.bat c:\buildkite-agent\hooks\
|
||||||
|
|
16
containers/buildkite-premerge-windows/cloudbuild.yaml
Normal file
16
containers/buildkite-premerge-windows/cloudbuild.yaml
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
#TODO replace network, subnetwork, region, zone and image with the placeholders. Trigger this from the terraform ?
|
||||||
|
#windows-server-2019-dc-core-for-containers-v20230113
|
||||||
|
#windows-server-2019-dc-for-containers-v20230113
|
||||||
|
#TODO get latest image in family
|
||||||
|
#TODO check different machine types and timings
|
||||||
|
steps:
|
||||||
|
- name: 'gcr.io/${PROJECT_ID}/windows-builder'
|
||||||
|
args: ['--image','windows-cloud/global/images/windows-server-2019-dc-core-for-containers-v20230113',
|
||||||
|
'--network', 'vpc-network',
|
||||||
|
'--subnetwork', 'subnetwork',
|
||||||
|
'--region', 'europe-west3',
|
||||||
|
'--zone', 'europe-west3-c',
|
||||||
|
'--machineType', "n2-standard-16",
|
||||||
|
'--diskType', 'pd-ssd',
|
||||||
|
'--command', 'gcloud auth configure-docker --quiet && docker build -t gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest . && docker push gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest']
|
||||||
|
timeout: 7200s
|
|
@ -1,35 +1,53 @@
|
||||||
ToDo:
|
#ToDo:
|
||||||
Move secrets to gcp secrets
|
Move secrets to gcp secrets
|
||||||
Add readme
|
Format readme in md
|
||||||
[]Add dependencies for kubernetes https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775
|
|
||||||
Cloud build for terraform
|
Cloud build for terraform
|
||||||
Check billing alerts
|
|
||||||
Push images to the cluster or to central location
|
|
||||||
[?]Wait for kubernetes plugin to finish apply
|
[?]Wait for kubernetes plugin to finish apply
|
||||||
|
Add readme if the budget is exceeded
|
||||||
|
|
||||||
1st run (bootstrap)
|
Be aware, the actions you execute on your gcp project will generate some cost.
|
||||||
|
|
||||||
Copy variables.tfvars from variables.tfvars_example
|
#Permissions
|
||||||
Insert project-id and billing-account
|
TODO
|
||||||
Insert secret values in the variables.tfvars file or insert values on runtime when using terraform apply
|
|
||||||
|
#1st run (bootstrap)
|
||||||
|
|
||||||
|
Copy `variables.tfvars` from `variables.tfvars_example`
|
||||||
|
Replace the placeholders for `project-id` and `billing-account` in `variables.tfvars`
|
||||||
|
Insert secret values in the `variables.tfvars` file or insert values on runtime when using terraform plan or apply
|
||||||
|
|
||||||
Initialise terraform
|
Initialise terraform
|
||||||
Comment out everything in backend.tf file to use local state
|
Comment out everything in `backend.tf` file to use local state for the first run as the bucket for storing the state is not created.
|
||||||
terraform init
|
```terraform init```
|
||||||
|
|
||||||
Create the state bucket
|
Create the state bucket
|
||||||
terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"
|
```terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"```
|
||||||
To disable the conformation use --auto-aprove flag
|
To disable the conformation use `--auto-aprove` flag
|
||||||
|
|
||||||
Move the state to the bucket
|
##Move the state to the bucket.
|
||||||
Uncomment everything in backend.tf file to use remote state
|
Uncomment everything in `backend.tf` file to use remote state with newly created bucket.
|
||||||
export PROJECT_ID="<PROJECT_ID>"
|
```export PROJECT_ID="<PROJECT_ID>"```
|
||||||
terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"
|
```terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"```
|
||||||
|
|
||||||
Create the cluster. Due to the problem described here [https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775] kubernetes provider requires cluster to be created. So we have to do the apply in 2 runs using the -target flag
|
Create the cluster. Due to the problem described [here](https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775) terraform kubernetes provider requires kubernetes cluster to be created first. So to create the cluster without applying kubernetes resources we will do the apply in 2 runs using the `-target` flag.
|
||||||
terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"
|
```terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"```
|
||||||
|
|
||||||
To deploy build slaves you need to have slaves docker images in your project. (TODO or we'll move them to the central project)
|
##Creating worker images
|
||||||
|
To deploy build workers you need the worker docker images in your project.
|
||||||
|
TODO cloud build SA permissions
|
||||||
|
|
||||||
Create the rest of the resources
|
###Linux worker image
|
||||||
terraform apply -var-file="variables.tfvars"
|
Execute cloud build to build Linux worker:
|
||||||
|
```gcloud builds submit --config=containers/buildkite-premerge-debian/cloudbuild.yaml containers/buildkite-premerge-debian/ --project=${PROJECT_ID}```
|
||||||
|
|
||||||
|
###Windows worker image
|
||||||
|
Build windows cloud builder. Follow the steps described here: [link](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/windows-builder)
|
||||||
|
|
||||||
|
Execute cloud build to build Windows worker:
|
||||||
|
```gcloud builds submit --config=containers/buildkite-premerge-windows/cloudbuild.yaml containers/buildkite-premerge-windows/ --project=${PROJECT_ID}```
|
||||||
|
|
||||||
|
##Create the rest of the gcp resources including workers in kubernetes pods
|
||||||
|
```terraform apply -var-file="variables.tfvars"```
|
||||||
|
|
||||||
|
#Budget
|
||||||
|
TODO
|
|
@ -34,7 +34,7 @@ spec:
|
||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: buildkite-premerge-debian
|
- name: buildkite-premerge-debian
|
||||||
image: gcr.io/${project-id}/buildkite-premerge-debian
|
image: gcr.io/${project-id}/buildkite-premerge-debian:latest
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
cpu: ${cpu-request}
|
cpu: ${cpu-request}
|
||||||
|
|
|
@ -34,7 +34,7 @@ spec:
|
||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: buildkite-premerge-windows
|
- name: buildkite-premerge-windows
|
||||||
image: gcr.io/${project-id}/buildkite-premerge-windows
|
image: gcr.io/${project-id}/buildkite-premerge-windows:latest
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
cpu: ${cpu-request}
|
cpu: ${cpu-request}
|
||||||
|
|
|
@ -1,5 +1,36 @@
|
||||||
#todo automatically rebuild buildkite images
|
#todo automatically rebuild buildkite images
|
||||||
|
|
||||||
|
data "google_project" "current_project" {
|
||||||
|
project_id = var.project-id
|
||||||
|
}
|
||||||
|
|
||||||
|
locals {
|
||||||
|
cloud_build_sa_roles = ["roles/storage.objectAdmin", "roles/compute.instanceAdmin", "roles/compute.securityAdmin"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# data "google_iam_policy" "cloud_build_sa" {
|
||||||
|
# binding {
|
||||||
|
# role = "roles/iam.serviceAccountUser"
|
||||||
|
|
||||||
|
# members = [
|
||||||
|
# "serviceAccount:${data.google_project.current_project.number}-compute@developer.gserviceaccount.com",
|
||||||
|
# ]
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
|
||||||
|
# resource "google_service_account_iam_policy" "admin-account-iam" {
|
||||||
|
# service_account_id = "${data.google_project.current_project.id}/serviceAccounts/${data.google_project.current_project.number}@cloudbuild.gserviceaccount.com"
|
||||||
|
# policy_data = data.google_iam_policy.cloud_build_sa.policy_data
|
||||||
|
# }
|
||||||
|
|
||||||
|
resource "google_project_iam_member" "cloudbuild_sa_roles" {
|
||||||
|
project = var.project-id
|
||||||
|
for_each = toset(local.cloud_build_sa_roles)
|
||||||
|
role = each.value
|
||||||
|
|
||||||
|
member = "serviceAccount:${data.google_project.current_project.number}@cloudbuild.gserviceaccount.com"
|
||||||
|
}
|
||||||
|
|
||||||
resource "google_project_service" "cloudbuild_api" {
|
resource "google_project_service" "cloudbuild_api" {
|
||||||
service = "cloudbuild.googleapis.com"
|
service = "cloudbuild.googleapis.com"
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,17 +3,17 @@ billing-account = ""
|
||||||
billing-budget = 25000
|
billing-budget = 25000
|
||||||
billing-admins = {"test": "test@test.com"}
|
billing-admins = {"test": "test@test.com"}
|
||||||
|
|
||||||
linux-agents-machine-type = "e2-standard-8"
|
#linux-agents-machine-type = "e2-standard-8"
|
||||||
linux-agents-count = 1
|
#linux-agents-count = 1
|
||||||
linux-agents-build-queue = "linux-test"
|
#linux-agents-build-queue = "linux-test"
|
||||||
linux-agents-cpu-request = "6"
|
#linux-agents-cpu-request = "6"
|
||||||
linux-agents-mem-request = "8Gi"
|
#linux-agents-mem-request = "8Gi"
|
||||||
|
|
||||||
windows-agents-machine-type = "e2-standard-8"
|
#windows-agents-machine-type = "e2-standard-8"
|
||||||
windows-agents-count = 1
|
#windows-agents-count = 1
|
||||||
windows-agents-build-queue = "windows-test"
|
#windows-agents-build-queue = "windows-test"
|
||||||
windows-agents-cpu-request = "6"
|
#windows-agents-cpu-request = "6"
|
||||||
windows-agents-mem-request = "8Gi"
|
#windows-agents-mem-request = "8Gi"
|
||||||
|
|
||||||
buildkite-api-token-readonly = ""
|
buildkite-api-token-readonly = ""
|
||||||
buildkite-agent-token = ""
|
buildkite-agent-token = ""
|
||||||
|
|
Loading…
Reference in a new issue