diff --git a/.gitignore b/.gitignore index 66fb502..042c0f8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,41 @@ containers/workspace **/.ipynb_checkpoints scripts/metrics/cache.json Pipfile.lock + +# Local .terraform directories +**/.terraform/* + +# .tflock files +.terraform.lock.hcl + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/terraform/backend.tf b/terraform/backend.tf new file mode 100644 index 0000000..2bffe49 --- /dev/null +++ b/terraform/backend.tf @@ -0,0 +1,7 @@ +terraform { + backend "gcs" { + #has to have the same name as the tf state bucket created in main.tf + bucket = "terraform-state-pre-merge-checks" #todo var + prefix = "terraform/state" + } +} \ No newline at end of file diff --git a/terraform/billing.tf b/terraform/billing.tf new file mode 100644 index 0000000..007e566 --- /dev/null +++ b/terraform/billing.tf @@ -0,0 +1,47 @@ +# #todo fix billing alert creation +# data "google_billing_account" "account" { +# billing_account = "01E34D-BF37C6-8137F6" +# } + +# resource "google_billing_budget" "budget" { +# billing_account = data.google_billing_account.account.id +# display_name = "budget" +# amount { +# specified_amount { +# currency_code = "USD" +# units = "25000" +# } +# } + +# budget_filter { +# #projects = ["projects/${data.google_project.project.number}"] +# credit_types_treatment = "EXCLUDE_ALL_CREDITS" +# #services = ["services/24E6-581D-38E5"] # Bigquery +# } + +# threshold_rules { +# threshold_percent = 0.5 +# } +# threshold_rules { +# threshold_percent = 0.9 +# } +# threshold_rules { +# threshold_percent = 1.0 +# } + +# # all_updates_rule { +# # monitoring_notification_channels = [ +# # google_monitoring_notification_channel.notification_channel.id, +# # ] +# # disable_default_iam_recipients = true +# # } +# } + +# resource "google_monitoring_notification_channel" "notification_channel" { +# display_name = "Example Notification Channel" +# type = "email" + +# labels = { +# email_address = "address@example.com" +# } +# } \ No newline at end of file diff --git a/terraform/cluster.tf b/terraform/cluster.tf new file mode 100644 index 0000000..ad3c7dd --- /dev/null +++ b/terraform/cluster.tf @@ -0,0 +1,94 @@ +resource "google_service_account" "llvm_premerge_checks_sa" { + account_id = "llvm-premerge-checks-sa" + display_name = "Service Account used with the gke cluster" +} + +resource "google_container_cluster" "llvm_premerge_checks_cluster" { + name = "llvm-premerge-checks-cluster" + + location = var.zone + + network = google_compute_network.vpc_network.name + subnetwork = google_compute_subnetwork.vpc_subnetwork.name + + #enable_autopilot = true + initial_node_count = 1 + + #TODO: redo + # master_authorized_networks_config { + # cidr_blocks { + # cidr_block= "0.0.0.0/0" + # display_name = "everyone" + # } + # } + + private_cluster_config { + enable_private_nodes = true + enable_private_endpoint = false + master_ipv4_cidr_block = var.master-cidr #todo: var + } + ip_allocation_policy { + cluster_secondary_range_name = "pods" + services_secondary_range_name = "services" + } +} + +resource "google_container_node_pool" "linux_agents_nodepool" { + name = "linux-agents" + cluster = google_container_cluster.llvm_premerge_checks_cluster.id + + node_config { + machine_type = var.linux-agents-machine-type + + #todo: assign right permissions and use custom service account + service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + } + + autoscaling { + min_node_count = 0 + max_node_count = 6 + } +} + +#todo recheck +data "google_client_config" "provider" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.llvm_premerge_checks_cluster.endpoint}" + token = data.google_client_config.provider.access_token + cluster_ca_certificate = base64decode( + google_container_cluster.llvm_premerge_checks_cluster.master_auth[0].cluster_ca_certificate, + ) +} + +resource "kubernetes_manifest" "buildkite_namespace" { + manifest = yamldecode(templatefile("kubernetes/namespace.yaml", {})) +} + +resource "kubernetes_manifest" "buildkite_agent_token_secret" { + manifest = yamldecode(templatefile("kubernetes/secret-buildkite-token.yaml", { buildkite-agent-token = var.buildkite-agent-token })) + depends_on = [kubernetes_manifest.buildkite_namespace] +} + +resource "kubernetes_manifest" "buildkite_api_token_readonly_secret" { + manifest = yamldecode(templatefile("kubernetes/secret-buildkite-token-readonly.yaml", { buildkite-api-token-readonly = var.buildkite-api-token-readonly })) + depends_on = [kubernetes_manifest.buildkite_namespace] +} + +resource "kubernetes_manifest" "buildkite_github_secret" { + manifest = yamldecode(templatefile("kubernetes/secret-github-ssh.yaml", { git-id-rsa = var.git-id-rsa, id-rsa-pub = var.id-rsa-pub, git-known-hosts = var.git-known-hosts })) + depends_on = [kubernetes_manifest.buildkite_namespace] +} + +resource "kubernetes_manifest" "buildkite_conduit_api_token_secret" { + manifest = yamldecode(templatefile("kubernetes/secret-conduit-token.yaml", { conduit-api-token = var.conduit-api-token })) + depends_on = [kubernetes_manifest.buildkite_namespace] +} + +resource "kubernetes_manifest" "buildkite_linux_agent" { + manifest = yamldecode(templatefile("kubernetes/linux-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.linux_agents_nodepool.name, build-queue = var.linux-agents-build-queue, cpu-request = var.linux-agents-cpu-request, mem-request = var.linux-agents-mem-request, replicas-count = var.linux-agents-count })) + depends_on = [kubernetes_manifest.buildkite_namespace] +} \ No newline at end of file diff --git a/terraform/kubernetes/linux-agents.yaml b/terraform/kubernetes/linux-agents.yaml new file mode 100644 index 0000000..78751d2 --- /dev/null +++ b/terraform/kubernetes/linux-agents.yaml @@ -0,0 +1,82 @@ +# Copyright 2021 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: linux-agents + namespace: buildkite +spec: + replicas: ${replicas-count} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + type: RollingUpdate + selector: + matchLabels: + app: agent-premerge-debian + template: + metadata: + labels: + app: agent-premerge-debian + spec: + containers: + - name: buildkite-premerge-debian + image: gcr.io/${project-id}/buildkite-premerge-debian + resources: + limits: + cpu: ${cpu-request} + memory: ${mem-request} + requests: + cpu: ${cpu-request} + memory: ${mem-request} + volumeMounts: + - name: github-ssh + mountPath: /mnt/ssh + - name: workdir + mountPath: /var/lib/buildkite-agent + env: + - name: BUILDKITE_AGENT_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-agent-token + key: token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: BUILDKITE_AGENT_TAGS + value: "queue=${build-queue},name=$(POD_NAME)" + - name: BUILDKITE_BUILD_PATH + value: "/var/lib/buildkite-agent/builds" + - name: CONDUIT_TOKEN + valueFrom: + secretKeyRef: + name: conduit-api-token + key: token + - name: BUILDKITE_API_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-api-token-readonly + key: token + volumes: + - name: github-ssh + secret: + secretName: github-ssh + - name: workdir + emptyDir: {} + nodeSelector: + cloud.google.com/gke-nodepool: ${gke-nodepool} #generate dynamically + terminationGracePeriodSeconds: 3600 \ No newline at end of file diff --git a/terraform/kubernetes/namespace.yaml b/terraform/kubernetes/namespace.yaml new file mode 100644 index 0000000..70c9001 --- /dev/null +++ b/terraform/kubernetes/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: buildkite \ No newline at end of file diff --git a/terraform/kubernetes/secret-buildkite-token-readonly.yaml b/terraform/kubernetes/secret-buildkite-token-readonly.yaml new file mode 100644 index 0000000..ab1b4f4 --- /dev/null +++ b/terraform/kubernetes/secret-buildkite-token-readonly.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: buildkite-api-token-readonly + namespace: buildkite +type: Opaque +data: + token: ${buildkite-api-token-readonly} \ No newline at end of file diff --git a/terraform/kubernetes/secret-buildkite-token.yaml b/terraform/kubernetes/secret-buildkite-token.yaml new file mode 100644 index 0000000..04182f2 --- /dev/null +++ b/terraform/kubernetes/secret-buildkite-token.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: buildkite-agent-token + namespace: buildkite +type: Opaque +data: + token: ${buildkite-agent-token} \ No newline at end of file diff --git a/terraform/kubernetes/secret-conduit-token.yaml b/terraform/kubernetes/secret-conduit-token.yaml new file mode 100644 index 0000000..b2be1ae --- /dev/null +++ b/terraform/kubernetes/secret-conduit-token.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: conduit-api-token + namespace: buildkite +type: Opaque +data: + token: ${conduit-api-token} \ No newline at end of file diff --git a/terraform/kubernetes/secret-github-ssh.yaml b/terraform/kubernetes/secret-github-ssh.yaml new file mode 100644 index 0000000..0067ec2 --- /dev/null +++ b/terraform/kubernetes/secret-github-ssh.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: github-ssh + namespace: buildkite +type: Opaque +data: + id_rsa: ${git-id-rsa} + id_rsa.pub: ${id-rsa-pub} + known_hosts: ${git-known-hosts} \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..60e5835 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,48 @@ +#todo automatically rebuild buildkite images + +resource "google_project_service" "cloudbuild_api" { + service = "cloudbuild.googleapis.com" +} + +resource "google_storage_bucket" "terraform_state" { + name = "terraform-state-${var.project-id}" #todo var + location = "EU" +} + +resource "google_compute_network" "vpc_network" { + name = "vpc-network" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "vpc_subnetwork" { + name = "subnetwork" + ip_cidr_range = var.subnetwork-main-cidr + region = var.region + network = google_compute_network.vpc_network.id + secondary_ip_range { + range_name = "pods" + ip_cidr_range = var.subnetwork-pods-cidr + } + secondary_ip_range { + range_name = "services" + ip_cidr_range = var.subnetwork-services-cidr + } +} + +resource "google_compute_router" "router" { + name = "router" + region = google_compute_subnetwork.vpc_subnetwork.region + network = google_compute_network.vpc_network.id + + bgp { + asn = 64514 + } +} + +resource "google_compute_router_nat" "nat" { + name = "router-nat" + router = google_compute_router.router.name + region = google_compute_router.router.region + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" +} diff --git a/terraform/provider.tf b/terraform/provider.tf new file mode 100644 index 0000000..ad8daba --- /dev/null +++ b/terraform/provider.tf @@ -0,0 +1,5 @@ +provider "google" { + project = var.project-id + region = var.region + zone = var.zone +} \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..f9e291e --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,82 @@ +variable "project-id" { + type = string +} + +variable "region" { + type = string + default = "europe-west3" +} + +variable "zone" { + type = string + default = "europe-west3-c" +} + +variable "subnetwork-main-cidr" { + type = string + default = "10.0.0.0/16" +} + +variable "master-cidr" { + type = string + default = "10.1.0.0/28" +} + +variable "subnetwork-pods-cidr" { + type = string + default = "10.2.0.0/16" +} + +variable "subnetwork-services-cidr" { + type = string + default = "10.3.0.0/16" +} + +variable "linux-agents-machine-type" { + type = string + default = "e2-standard-32" +} + +variable "linux-agents-count" { + type = number + default = 6 +} + +variable "linux-agents-build-queue" { + type = string + default = "linux" +} + +variable "linux-agents-cpu-request" { + type = string + default = "30" +} + +variable "linux-agents-mem-request" { + type = string + default = "80Gi" +} + +variable "buildkite-api-token-readonly" { + type = string +} + +variable "buildkite-agent-token" { + type = string +} + +variable "conduit-api-token" { + type = string +} + +variable "git-id-rsa" { + type = string +} + +variable "id-rsa-pub" { + type = string +} + +variable "git-known-hosts" { + type = string +}