diff --git a/containers/buildkite-premerge-debian/.dockerignore b/containers/buildkite-premerge-debian/.dockerignore new file mode 100644 index 0000000..74bf493 --- /dev/null +++ b/containers/buildkite-premerge-debian/.dockerignore @@ -0,0 +1 @@ +cloudbuild.yaml \ No newline at end of file diff --git a/containers/buildkite-premerge-debian/cloudbuild.yaml b/containers/buildkite-premerge-debian/cloudbuild.yaml new file mode 100644 index 0000000..cf2c0b7 --- /dev/null +++ b/containers/buildkite-premerge-debian/cloudbuild.yaml @@ -0,0 +1,6 @@ +steps: + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/${PROJECT_ID}/buildkite-premerge-debian:latest', '.'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/${PROJECT_ID}/buildkite-premerge-debian:latest'] \ No newline at end of file diff --git a/containers/buildkite-premerge-windows/.dockerignore b/containers/buildkite-premerge-windows/.dockerignore new file mode 100644 index 0000000..74bf493 --- /dev/null +++ b/containers/buildkite-premerge-windows/.dockerignore @@ -0,0 +1 @@ +cloudbuild.yaml \ No newline at end of file diff --git a/containers/buildkite-premerge-windows/Dockerfile b/containers/buildkite-premerge-windows/Dockerfile index e187304..4183ff4 100644 --- a/containers/buildkite-premerge-windows/Dockerfile +++ b/containers/buildkite-premerge-windows/Dockerfile @@ -93,7 +93,7 @@ RUN powershell -Command ` RUN git config --system core.longpaths true & ` git config --global core.autocrlf false -# For debugging of files beeing locked by some processes. +# handle for debugging of files beeing locked by some processes. RUN choco install -y handle COPY start_agent.ps1 C:\scripts\ COPY pre-checkout.bat c:\buildkite-agent\hooks\ diff --git a/containers/buildkite-premerge-windows/cloudbuild.yaml b/containers/buildkite-premerge-windows/cloudbuild.yaml new file mode 100644 index 0000000..2e6b3bd --- /dev/null +++ b/containers/buildkite-premerge-windows/cloudbuild.yaml @@ -0,0 +1,16 @@ +#TODO replace network, subnetwork, region, zone and image with the placeholders. Trigger this from the terraform ? +#windows-server-2019-dc-core-for-containers-v20230113 +#windows-server-2019-dc-for-containers-v20230113 +#TODO get latest image in family +#TODO check different machine types and timings +steps: + - name: 'gcr.io/${PROJECT_ID}/windows-builder' + args: ['--image','windows-cloud/global/images/windows-server-2019-dc-core-for-containers-v20230113', + '--network', 'vpc-network', + '--subnetwork', 'subnetwork', + '--region', 'europe-west3', + '--zone', 'europe-west3-c', + '--machineType', "n2-standard-16", + '--diskType', 'pd-ssd', + '--command', 'gcloud auth configure-docker --quiet && docker build -t gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest . && docker push gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest'] +timeout: 7200s \ No newline at end of file diff --git a/terraform/README.md b/terraform/README.md index efb10d2..6fdd74c 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,35 +1,53 @@ -ToDo: +#ToDo: Move secrets to gcp secrets -Add readme -[]Add dependencies for kubernetes https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775 +Format readme in md Cloud build for terraform -Check billing alerts -Push images to the cluster or to central location [?]Wait for kubernetes plugin to finish apply +Add readme if the budget is exceeded -1st run (bootstrap) +Be aware, the actions you execute on your gcp project will generate some cost. -Copy variables.tfvars from variables.tfvars_example -Insert project-id and billing-account -Insert secret values in the variables.tfvars file or insert values on runtime when using terraform apply +#Permissions +TODO + +#1st run (bootstrap) + +Copy `variables.tfvars` from `variables.tfvars_example` +Replace the placeholders for `project-id` and `billing-account` in `variables.tfvars` +Insert secret values in the `variables.tfvars` file or insert values on runtime when using terraform plan or apply Initialise terraform -Comment out everything in backend.tf file to use local state -terraform init +Comment out everything in `backend.tf` file to use local state for the first run as the bucket for storing the state is not created. +```terraform init``` Create the state bucket -terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state" -To disable the conformation use --auto-aprove flag +```terraform apply -var-file=variables.tfvars -target="google_storage_bucket.terraform_state"``` +To disable the conformation use `--auto-aprove` flag -Move the state to the bucket -Uncomment everything in backend.tf file to use remote state -export PROJECT_ID="" -terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state" +##Move the state to the bucket. +Uncomment everything in `backend.tf` file to use remote state with newly created bucket. +```export PROJECT_ID=""``` +```terraform init -backend-config="bucket=terraform-state-${PROJECT_ID}" -backend-config="prefix=terraform/state"``` -Create the cluster. Due to the problem described here [https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775] kubernetes provider requires cluster to be created. So we have to do the apply in 2 runs using the -target flag -terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster" +Create the cluster. Due to the problem described [here](https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775) terraform kubernetes provider requires kubernetes cluster to be created first. So to create the cluster without applying kubernetes resources we will do the apply in 2 runs using the `-target` flag. +```terraform apply -var-file=variables.tfvars -target="google_container_cluster.llvm_premerge_checks_cluster"``` -To deploy build slaves you need to have slaves docker images in your project. (TODO or we'll move them to the central project) +##Creating worker images +To deploy build workers you need the worker docker images in your project. +TODO cloud build SA permissions -Create the rest of the resources -terraform apply -var-file="variables.tfvars" \ No newline at end of file +###Linux worker image +Execute cloud build to build Linux worker: +```gcloud builds submit --config=containers/buildkite-premerge-debian/cloudbuild.yaml containers/buildkite-premerge-debian/ --project=${PROJECT_ID}``` + +###Windows worker image +Build windows cloud builder. Follow the steps described here: [link](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/windows-builder) + +Execute cloud build to build Windows worker: +```gcloud builds submit --config=containers/buildkite-premerge-windows/cloudbuild.yaml containers/buildkite-premerge-windows/ --project=${PROJECT_ID}``` + +##Create the rest of the gcp resources including workers in kubernetes pods +```terraform apply -var-file="variables.tfvars"``` + +#Budget +TODO \ No newline at end of file diff --git a/terraform/kubernetes/linux-agents.yaml b/terraform/kubernetes/linux-agents.yaml index a3b37b6..25db26b 100644 --- a/terraform/kubernetes/linux-agents.yaml +++ b/terraform/kubernetes/linux-agents.yaml @@ -34,7 +34,7 @@ spec: spec: containers: - name: buildkite-premerge-debian - image: gcr.io/${project-id}/buildkite-premerge-debian + image: gcr.io/${project-id}/buildkite-premerge-debian:latest resources: limits: cpu: ${cpu-request} diff --git a/terraform/kubernetes/windows-agents.yaml b/terraform/kubernetes/windows-agents.yaml index 740cdc7..6eb86e5 100644 --- a/terraform/kubernetes/windows-agents.yaml +++ b/terraform/kubernetes/windows-agents.yaml @@ -34,7 +34,7 @@ spec: spec: containers: - name: buildkite-premerge-windows - image: gcr.io/${project-id}/buildkite-premerge-windows + image: gcr.io/${project-id}/buildkite-premerge-windows:latest resources: limits: cpu: ${cpu-request} diff --git a/terraform/main.tf b/terraform/main.tf index 314f35b..563e7e4 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,5 +1,36 @@ #todo automatically rebuild buildkite images +data "google_project" "current_project" { + project_id = var.project-id +} + +locals { + cloud_build_sa_roles = ["roles/storage.objectAdmin", "roles/compute.instanceAdmin", "roles/compute.securityAdmin"] +} + +# data "google_iam_policy" "cloud_build_sa" { +# binding { +# role = "roles/iam.serviceAccountUser" + +# members = [ +# "serviceAccount:${data.google_project.current_project.number}-compute@developer.gserviceaccount.com", +# ] +# } +# } + +# resource "google_service_account_iam_policy" "admin-account-iam" { +# service_account_id = "${data.google_project.current_project.id}/serviceAccounts/${data.google_project.current_project.number}@cloudbuild.gserviceaccount.com" +# policy_data = data.google_iam_policy.cloud_build_sa.policy_data +# } + +resource "google_project_iam_member" "cloudbuild_sa_roles" { + project = var.project-id + for_each = toset(local.cloud_build_sa_roles) + role = each.value + + member = "serviceAccount:${data.google_project.current_project.number}@cloudbuild.gserviceaccount.com" +} + resource "google_project_service" "cloudbuild_api" { service = "cloudbuild.googleapis.com" } diff --git a/terraform/variables.tfvars_example b/terraform/variables.tfvars_example index d831fdd..d0a2daf 100644 --- a/terraform/variables.tfvars_example +++ b/terraform/variables.tfvars_example @@ -3,17 +3,17 @@ billing-account = "" billing-budget = 25000 billing-admins = {"test": "test@test.com"} -linux-agents-machine-type = "e2-standard-8" -linux-agents-count = 1 -linux-agents-build-queue = "linux-test" -linux-agents-cpu-request = "6" -linux-agents-mem-request = "8Gi" +#linux-agents-machine-type = "e2-standard-8" +#linux-agents-count = 1 +#linux-agents-build-queue = "linux-test" +#linux-agents-cpu-request = "6" +#linux-agents-mem-request = "8Gi" -windows-agents-machine-type = "e2-standard-8" -windows-agents-count = 1 -windows-agents-build-queue = "windows-test" -windows-agents-cpu-request = "6" -windows-agents-mem-request = "8Gi" +#windows-agents-machine-type = "e2-standard-8" +#windows-agents-count = 1 +#windows-agents-build-queue = "windows-test" +#windows-agents-cpu-request = "6" +#windows-agents-mem-request = "8Gi" buildkite-api-token-readonly = "" buildkite-agent-token = ""