Add terraform configuration

.gitignore

@@ -8,3 +8,41 @@ containers/workspace
 **/.ipynb_checkpoints
 scripts/metrics/cache.json
 Pipfile.lock
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tflock files
+.terraform.lock.hcl
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

terraform/backend.tf

@@ -0,0 +1,7 @@
+terraform {
+  backend "gcs" {
+    #has to have the same name as the tf state bucket created in main.tf
+    bucket = "terraform-state-pre-merge-checks" #todo var
+    prefix = "terraform/state"
+  }
+}

terraform/billing.tf

@@ -0,0 +1,47 @@
+# #todo fix billing alert creation
+# data "google_billing_account" "account" {
+#   billing_account = "01E34D-BF37C6-8137F6"
+# }
+
+# resource "google_billing_budget" "budget" {
+#   billing_account = data.google_billing_account.account.id
+#   display_name = "budget"
+#   amount {
+#     specified_amount {
+#       currency_code = "USD"
+#       units = "25000"
+#     }
+#   }
+
+#   budget_filter {
+#     #projects = ["projects/${data.google_project.project.number}"]
+#     credit_types_treatment = "EXCLUDE_ALL_CREDITS"
+#     #services = ["services/24E6-581D-38E5"] # Bigquery
+#   }
+
+#   threshold_rules {
+#     threshold_percent = 0.5
+#   }
+#   threshold_rules {
+#     threshold_percent = 0.9
+#   }
+#   threshold_rules {
+#     threshold_percent = 1.0
+#   }
+
+#   # all_updates_rule {
+#   #   monitoring_notification_channels = [
+#   #     google_monitoring_notification_channel.notification_channel.id,
+#   #   ]
+#   #   disable_default_iam_recipients = true
+#   # }
+# }
+
+# resource "google_monitoring_notification_channel" "notification_channel" {
+#   display_name = "Example Notification Channel"
+#   type         = "email"
+
+#   labels = {
+#     email_address = "address@example.com"
+#   }
+# }

terraform/cluster.tf

@@ -0,0 +1,94 @@
+resource "google_service_account" "llvm_premerge_checks_sa" {
+  account_id   = "llvm-premerge-checks-sa"
+  display_name = "Service Account used with the gke cluster"
+}
+
+resource "google_container_cluster" "llvm_premerge_checks_cluster" {
+  name = "llvm-premerge-checks-cluster"
+
+  location = var.zone
+
+  network    = google_compute_network.vpc_network.name
+  subnetwork = google_compute_subnetwork.vpc_subnetwork.name
+
+  #enable_autopilot = true
+  initial_node_count = 1
+
+  #TODO: redo
+  # master_authorized_networks_config {
+  #   cidr_blocks {
+  #     cidr_block= "0.0.0.0/0"
+  #     display_name = "everyone"
+  #   }
+  # }
+
+  private_cluster_config {
+    enable_private_nodes    = true
+    enable_private_endpoint = false
+    master_ipv4_cidr_block  = var.master-cidr #todo: var
+  }
+  ip_allocation_policy {
+    cluster_secondary_range_name  = "pods"
+    services_secondary_range_name = "services"
+  }
+}
+
+resource "google_container_node_pool" "linux_agents_nodepool" {
+  name    = "linux-agents"
+  cluster = google_container_cluster.llvm_premerge_checks_cluster.id
+
+  node_config {
+    machine_type = var.linux-agents-machine-type
+
+    #todo: assign right permissions and use custom service account
+    service_account = "1047329282069-compute@developer.gserviceaccount.com" #google_service_account.llvm_premerge_checks_sa.email
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/cloud-platform"
+    ]
+  }
+
+  autoscaling {
+    min_node_count = 0
+    max_node_count = 6
+  }
+}
+
+#todo recheck
+data "google_client_config" "provider" {}
+
+provider "kubernetes" {
+  host  = "https://${google_container_cluster.llvm_premerge_checks_cluster.endpoint}"
+  token = data.google_client_config.provider.access_token
+  cluster_ca_certificate = base64decode(
+    google_container_cluster.llvm_premerge_checks_cluster.master_auth[0].cluster_ca_certificate,
+  )
+}
+
+resource "kubernetes_manifest" "buildkite_namespace" {
+  manifest = yamldecode(templatefile("kubernetes/namespace.yaml", {}))
+}
+
+resource "kubernetes_manifest" "buildkite_agent_token_secret" {
+  manifest   = yamldecode(templatefile("kubernetes/secret-buildkite-token.yaml", { buildkite-agent-token = var.buildkite-agent-token }))
+  depends_on = [kubernetes_manifest.buildkite_namespace]
+}
+
+resource "kubernetes_manifest" "buildkite_api_token_readonly_secret" {
+  manifest   = yamldecode(templatefile("kubernetes/secret-buildkite-token-readonly.yaml", { buildkite-api-token-readonly = var.buildkite-api-token-readonly }))
+  depends_on = [kubernetes_manifest.buildkite_namespace]
+}
+
+resource "kubernetes_manifest" "buildkite_github_secret" {
+  manifest   = yamldecode(templatefile("kubernetes/secret-github-ssh.yaml", { git-id-rsa = var.git-id-rsa, id-rsa-pub = var.id-rsa-pub, git-known-hosts = var.git-known-hosts }))
+  depends_on = [kubernetes_manifest.buildkite_namespace]
+}
+
+resource "kubernetes_manifest" "buildkite_conduit_api_token_secret" {
+  manifest   = yamldecode(templatefile("kubernetes/secret-conduit-token.yaml", { conduit-api-token = var.conduit-api-token }))
+  depends_on = [kubernetes_manifest.buildkite_namespace]
+}
+
+resource "kubernetes_manifest" "buildkite_linux_agent" {
+  manifest   = yamldecode(templatefile("kubernetes/linux-agents.yaml", { project-id = var.project-id, gke-nodepool = google_container_node_pool.linux_agents_nodepool.name, build-queue = var.linux-agents-build-queue, cpu-request = var.linux-agents-cpu-request, mem-request = var.linux-agents-mem-request, replicas-count = var.linux-agents-count }))
+  depends_on = [kubernetes_manifest.buildkite_namespace]
+}

terraform/kubernetes/linux-agents.yaml

@@ -0,0 +1,82 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: linux-agents
+  namespace: buildkite
+spec:
+  replicas: ${replicas-count}
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 50%
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: agent-premerge-debian
+  template:
+    metadata:
+      labels:
+        app: agent-premerge-debian
+    spec:
+      containers:
+        - name: buildkite-premerge-debian
+          image: gcr.io/${project-id}/buildkite-premerge-debian
+          resources:
+            limits:
+              cpu: ${cpu-request}
+              memory: ${mem-request}
+            requests:
+              cpu: ${cpu-request}
+              memory: ${mem-request}
+          volumeMounts:
+            - name: github-ssh
+              mountPath: /mnt/ssh
+            - name: workdir
+              mountPath: /var/lib/buildkite-agent
+          env:
+            - name: BUILDKITE_AGENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: buildkite-agent-token
+                  key: token
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                 fieldPath: metadata.name
+            - name: BUILDKITE_AGENT_TAGS
+              value: "queue=${build-queue},name=$(POD_NAME)"
+            - name: BUILDKITE_BUILD_PATH
+              value: "/var/lib/buildkite-agent/builds"
+            - name: CONDUIT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: conduit-api-token
+                  key: token
+            - name: BUILDKITE_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: buildkite-api-token-readonly
+                  key: token
+      volumes:
+        - name: github-ssh
+          secret:
+            secretName: github-ssh
+        - name: workdir
+          emptyDir: {}
+      nodeSelector:
+        cloud.google.com/gke-nodepool: ${gke-nodepool} #generate dynamically
+      terminationGracePeriodSeconds: 3600

terraform/kubernetes/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: buildkite

terraform/kubernetes/secret-buildkite-token-readonly.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: buildkite-api-token-readonly
+  namespace: buildkite
+type: Opaque
+data:
+  token: ${buildkite-api-token-readonly}

terraform/kubernetes/secret-buildkite-token.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: buildkite-agent-token
+  namespace: buildkite
+type: Opaque
+data:
+  token: ${buildkite-agent-token}

terraform/kubernetes/secret-conduit-token.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: conduit-api-token
+  namespace: buildkite
+type: Opaque
+data:
+  token: ${conduit-api-token}

terraform/kubernetes/secret-github-ssh.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: github-ssh
+  namespace: buildkite
+type: Opaque
+data:
+  id_rsa: ${git-id-rsa}
+  id_rsa.pub: ${id-rsa-pub}
+  known_hosts: ${git-known-hosts}

terraform/main.tf

@@ -0,0 +1,48 @@
+#todo automatically rebuild buildkite images
+
+resource "google_project_service" "cloudbuild_api" {
+  service = "cloudbuild.googleapis.com"
+}
+
+resource "google_storage_bucket" "terraform_state" {
+  name     = "terraform-state-${var.project-id}" #todo var
+  location = "EU"
+}
+
+resource "google_compute_network" "vpc_network" {
+  name                    = "vpc-network"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "vpc_subnetwork" {
+  name          = "subnetwork"
+  ip_cidr_range = var.subnetwork-main-cidr
+  region        = var.region
+  network       = google_compute_network.vpc_network.id
+  secondary_ip_range {
+    range_name    = "pods"
+    ip_cidr_range = var.subnetwork-pods-cidr
+  }
+  secondary_ip_range {
+    range_name    = "services"
+    ip_cidr_range = var.subnetwork-services-cidr
+  }
+}
+
+resource "google_compute_router" "router" {
+  name    = "router"
+  region  = google_compute_subnetwork.vpc_subnetwork.region
+  network = google_compute_network.vpc_network.id
+
+  bgp {
+    asn = 64514
+  }
+}
+
+resource "google_compute_router_nat" "nat" {
+  name                               = "router-nat"
+  router                             = google_compute_router.router.name
+  region                             = google_compute_router.router.region
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+}

terraform/provider.tf

@@ -0,0 +1,5 @@
+provider "google" {
+  project = var.project-id
+  region  = var.region
+  zone    = var.zone
+}

terraform/variables.tf

@@ -0,0 +1,82 @@
+variable "project-id" {
+  type = string
+}
+
+variable "region" {
+  type    = string
+  default = "europe-west3"
+}
+
+variable "zone" {
+  type    = string
+  default = "europe-west3-c"
+}
+
+variable "subnetwork-main-cidr" {
+  type    = string
+  default = "10.0.0.0/16"
+}
+
+variable "master-cidr" {
+  type    = string
+  default = "10.1.0.0/28"
+}
+
+variable "subnetwork-pods-cidr" {
+  type    = string
+  default = "10.2.0.0/16"
+}
+
+variable "subnetwork-services-cidr" {
+  type    = string
+  default = "10.3.0.0/16"
+}
+
+variable "linux-agents-machine-type" {
+  type    = string
+  default = "e2-standard-32"
+}
+
+variable "linux-agents-count" {
+  type    = number
+  default = 6
+}
+
+variable "linux-agents-build-queue" {
+  type    = string
+  default = "linux"
+}
+
+variable "linux-agents-cpu-request" {
+  type    = string
+  default = "30"
+}
+
+variable "linux-agents-mem-request" {
+  type    = string
+  default = "80Gi"
+}
+
+variable "buildkite-api-token-readonly" {
+  type = string
+}
+
+variable "buildkite-agent-token" {
+  type = string
+}
+
+variable "conduit-api-token" {
+  type = string
+}
+
+variable "git-id-rsa" {
+  type = string
+}
+
+variable "id-rsa-pub" {
+  type = string
+}
+
+variable "git-known-hosts" {
+  type = string
+}