From dd96ebb61daeaa60da65fada13c8439dad4bca25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=BChnel?= Date: Fri, 4 Oct 2019 10:26:20 +0200 Subject: [PATCH] initial import of configuration files --- .gitignore | 1 + README.md | 84 +++++++++ containers/README.md | 2 + .../agent-debian-testing-clang8/Dockerfile | 38 ++++ .../authorized_keys | 1 + .../build_deploy.sh | 15 ++ .../agent-debian-testing-clang8/build_run.sh | 7 + containers/jenkins-master/Dockerfile | 1 + containers/jenkins-master/build_deploy.sh | 17 ++ k8s_config | 5 + kubernetes/cluster_create.sh | 12 ++ kubernetes/cluster_delete.sh | 13 ++ kubernetes/deploy.sh | 4 + kubernetes/disk_create.sh | 14 ++ kubernetes/disk_delete.sh | 6 + kubernetes/jenkins.yaml | 171 ++++++++++++++++++ local_setup.sh | 16 ++ 17 files changed, 407 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 containers/README.md create mode 100644 containers/agent-debian-testing-clang8/Dockerfile create mode 100644 containers/agent-debian-testing-clang8/authorized_keys create mode 100755 containers/agent-debian-testing-clang8/build_deploy.sh create mode 100755 containers/agent-debian-testing-clang8/build_run.sh create mode 100644 containers/jenkins-master/Dockerfile create mode 100755 containers/jenkins-master/build_deploy.sh create mode 100644 k8s_config create mode 100755 kubernetes/cluster_create.sh create mode 100755 kubernetes/cluster_delete.sh create mode 100755 kubernetes/deploy.sh create mode 100755 kubernetes/disk_create.sh create mode 100755 kubernetes/disk_delete.sh create mode 100644 kubernetes/jenkins.yaml create mode 100755 local_setup.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dbe9c82 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8c665a8 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# Overview + +This repository contains the configuration files for the merge guards for the LLVM project. It configures a cluster of build machines that are used to check all incoming commits to the LLVM project. + +# Merge guards +TODO(@christiankuehnel): describe objective of merge guards + +# Cluster overview +TODO(@christiankuehnel): describe how the cluster is set up + +# Phabricator integration +TODO(@christiankuehnel): describe how this is integrated with Phabricator + +# Playbooks + +## deployment to a clean infrastructure +General remarks: +* GCP does not route any traffic to your services unless the service is "helthy". It might take a few minutes after startup before the services is classified as healthy. Until then you will only see some generic error message. + +These are the steps to set up the build server on a clean infrastructure: +1. Configure the tools on your local machine: + ```bash + ./setup.sh + ``` +1. Delete the old cluster, if it still exists: + ```bash + cd kubernetes/cluster + ./cluster_delete.sh + ``` +1. Create the cluster: + ```bash + cd kubernetes/cluster + ./cluster_create.sh + ``` +1. Create the disk storage, if it does not yet exist: + ```bash + cd kubernetes/cluster + ./disk_create.sh + ``` +1. ssh into the VM instance mounting the volume, find the mount point and then set + ```bash + # go to the mount point of the volume + cd /var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/jenkins-home + # change the permissions + sudo chmod a+rwx + ``` +1. push the docker images to gcr.io: +```bash + cd containers/debian-testing-clang8 + ./build_deploy.sh + + cd ../jenkins-master + ./build_deploy.sh +``` +1. Deploy the stack: `` + ```bash + cd kubernetes + ./deploy.sh + ``` +1. configure it + +## handling SSH keys +The Jenkins server SSHs into the agents to start the agent application. Thus the master needs SSH access to the agent. To set this up: + +1. Create an SSH key pair locally with `ssh-keygen`. +1. Copy the contents of `id_rsa` to the credentials section of the Jenkins UI. +1. Configure the agent in the Jenkins UI to use the new SSH keys you just uploaded. +1. Copy the contents of `id_rsa.pub` to `containers//authorized keys`. +1. Rebuild and deploy the agents. + +While this works, it does not fell like the perfect solution. I'm happy to get better ideas on this. + +# TODO list +* deploy Phabricator clone +* hook up first "clang-format" job +* deploy storage server, store build results there +* setup proper ingress server with SSL and user authentication +* Add missing configuration steps to these scripts + * deploy and recover jenkings master configuration +* set up DNS names for agents, so the master can address them by name, not by IP + +* change permissions to jenkins home folder by script + * needs to be done before launching the Jenkins binary + * probably add this to the jenkins.sh script in the docker container diff --git a/containers/README.md b/containers/README.md new file mode 100644 index 0000000..1f0fd27 --- /dev/null +++ b/containers/README.md @@ -0,0 +1,2 @@ +# Overview +This folder contains files related to the machines running the build/test/checks for the merge guards. diff --git a/containers/agent-debian-testing-clang8/Dockerfile b/containers/agent-debian-testing-clang8/Dockerfile new file mode 100644 index 0000000..c86a386 --- /dev/null +++ b/containers/agent-debian-testing-clang8/Dockerfile @@ -0,0 +1,38 @@ +FROM debian:testing +VOLUME ["/ccache"] + +RUN apt-get update ;\ + apt-get install -y --no-install-recommends \ + cmake ninja-build git ca-certificates clang-8 lld-8 ccache python python3 build-essential \ + clang-tidy-8 clang-format-8 \ + python-psutil arcanist \ + openjdk-11-jdk \ + openssh-server ;\ + apt-get clean + +# required for openssh server +RUN mkdir -p /run/sshd + +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN groupadd -g ${gid} ${group} ;\ + useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\ + mkdir /home/${user}/agent ; \ + mkdir /home/${user}/ccache + +COPY authorized_keys /home/${user}/.ssh/ + +RUN chown -R ${user}:${user} /home/${user} ;\ + chmod 700 /home/${user}/.ssh + +WORKDIR ${AGENT_WORKDIR} +ENV CC=clang-8 +ENV CXX=clang++-8 + +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] + diff --git a/containers/agent-debian-testing-clang8/authorized_keys b/containers/agent-debian-testing-clang8/authorized_keys new file mode 100644 index 0000000..06f2ccf --- /dev/null +++ b/containers/agent-debian-testing-clang8/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwYAn/XBFxk1uBhgYCuNLLpriM88dIa22nZdXjzdW7TjGMM2PbJQEPRvGRJjFkgxcRZgND6+oGD8ldMacI7TF5m+TF4TXEp7PKbjeP0vqWx1d/xn2QxDHItWNLXfV9F96TNCMIHjcV9iHZ9u4ciudw51NmsT7oVDsyy+lB9+/I7RbOTzCsDMQQcBdHTCXBqGSo7UtxkQ3qgsWPYDpQpBa8lSIUBhCnsG+hnjk1IHYu9XFqo5jzeUR4DzAuUqzZuxAb/dYWkPjpFlH8RSPpvxRGl3xXeH0VQ9yrlm8zqfTCmXSv14zRpGTrfQFrXXnJTKcQ4+5ACeXOnTKiiWhjlYT3 kuhnel@kuhnel.muc.corp.google.com diff --git a/containers/agent-debian-testing-clang8/build_deploy.sh b/containers/agent-debian-testing-clang8/build_deploy.sh new file mode 100755 index 0000000..cf045e0 --- /dev/null +++ b/containers/agent-debian-testing-clang8/build_deploy.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -eux + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname $(dirname ${DIR}))" + +# get config options +source "${ROOT_DIR}/k8s_config" + +IMAGE_NAME="agent-debian-testing-clang8" +QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}" + +docker build -t ${IMAGE_NAME} . +docker tag ${IMAGE_NAME} ${QUALIFIED_NAME} +docker push ${QUALIFIED_NAME} \ No newline at end of file diff --git a/containers/agent-debian-testing-clang8/build_run.sh b/containers/agent-debian-testing-clang8/build_run.sh new file mode 100755 index 0000000..209271f --- /dev/null +++ b/containers/agent-debian-testing-clang8/build_run.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eux + +IMAGE_NAME="agent-debian-testing-clang8" + +docker build -t ${IMAGE_NAME} . +docker run -i -t ${IMAGE_NAME} \ No newline at end of file diff --git a/containers/jenkins-master/Dockerfile b/containers/jenkins-master/Dockerfile new file mode 100644 index 0000000..cad1249 --- /dev/null +++ b/containers/jenkins-master/Dockerfile @@ -0,0 +1 @@ +FROM jenkins/jenkins:lts \ No newline at end of file diff --git a/containers/jenkins-master/build_deploy.sh b/containers/jenkins-master/build_deploy.sh new file mode 100755 index 0000000..a3d3110 --- /dev/null +++ b/containers/jenkins-master/build_deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -eux + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname $(dirname ${DIR}))" + +# get config options +source "${ROOT_DIR}/k8s_config" + +IMAGE_NAME="jenkins-master" +QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}" + +docker build -t ${IMAGE_NAME} . +docker tag ${IMAGE_NAME} ${QUALIFIED_NAME} +docker push ${QUALIFIED_NAME} + +echo "Deployed as ${QUALIFIED_NAME}." \ No newline at end of file diff --git a/k8s_config b/k8s_config new file mode 100644 index 0000000..029f731 --- /dev/null +++ b/k8s_config @@ -0,0 +1,5 @@ +# config options +GCP_ZONE="us-central1-a" +GCP_CLUSTER="llvm-merge-guards" +GCP_PROJECT="llvm-windows-development" +GCR_HOSTNAME="gcr.io" diff --git a/kubernetes/cluster_create.sh b/kubernetes/cluster_create.sh new file mode 100755 index 0000000..cb02ff1 --- /dev/null +++ b/kubernetes/cluster_create.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -eux + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname ${DIR})" + +# get config options +source "${ROOT_DIR}/k8s_config" + +# create the cluster +gcloud container clusters create $GCP_CLUSTER --zone $GCP_ZONE \ + --machine-type=n1-standard-32 --num-nodes=1 diff --git a/kubernetes/cluster_delete.sh b/kubernetes/cluster_delete.sh new file mode 100755 index 0000000..c7025b0 --- /dev/null +++ b/kubernetes/cluster_delete.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -eux + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname ${DIR})" + +# get config options +source "${ROOT_DIR}/k8s_config" + +#TODO: check gcloud auth login + +# create the cluster +gcloud container clusters delete $GCP_CLUSTER \ No newline at end of file diff --git a/kubernetes/deploy.sh b/kubernetes/deploy.sh new file mode 100755 index 0000000..14d92d0 --- /dev/null +++ b/kubernetes/deploy.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -eux + +kubectl apply -f jenkins.yaml \ No newline at end of file diff --git a/kubernetes/disk_create.sh b/kubernetes/disk_create.sh new file mode 100755 index 0000000..812eba1 --- /dev/null +++ b/kubernetes/disk_create.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -eux + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname ${DIR})" + +# get config options +source "${ROOT_DIR}/k8s_config" + +gcloud compute disks create jenkins-home \ + --description="storage for jenkins master" \ + --size=200GB \ + --type=pd-standard \ + --zone=${GCP_ZONE} \ \ No newline at end of file diff --git a/kubernetes/disk_delete.sh b/kubernetes/disk_delete.sh new file mode 100755 index 0000000..0e15943 --- /dev/null +++ b/kubernetes/disk_delete.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -eux + +# to detatch: +# gcloud compute instances detach-disk --disk=jenkins-home +gcloud compute disks delete jenkins-home \ No newline at end of file diff --git a/kubernetes/jenkins.yaml b/kubernetes/jenkins.yaml new file mode 100644 index 0000000..512c7d4 --- /dev/null +++ b/kubernetes/jenkins.yaml @@ -0,0 +1,171 @@ +# from https://cloud.google.com/solutions/jenkins-on-kubernetes-engine +# and https://github.com/GoogleCloudPlatform/continuous-deployment-on-kubernetes/tree/v1/jenkins/k8s + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: jenkins + labels: + name: jenkins +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default + namespace: jenkins +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: jenkins-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: default + namespace: jenkins +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: jenkins-home + labels: + failure-domain.beta.kubernetes.io/zone: us-central1-a +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + gcePersistentDisk: + pdName: jenkins-home + fsType: ext4 +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: jenkins + namespace: jenkins +spec: + replicas: 1 + template: + metadata: + labels: + app: master + spec: + hostname: jenkins-master + containers: + - name: master + image: gcr.io/llvm-windows-development/jenkins-master + ports: + - containerPort: 8080 + - containerPort: 50000 + readinessProbe: + httpGet: + path: /login + port: 8080 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 2 + failureThreshold: 5 + env: + - name: JAVA_OPTS + value: '-Xmx1400m' + volumeMounts: + - mountPath: /var/jenkins_home + name: jenkins-home + resources: + limits: + cpu: 500m + memory: 1500Mi + requests: + cpu: 500m + memory: 1500Mi + volumes: + - name: jenkins-home + gcePersistentDisk: + pdName: jenkins-home + fsType: ext4 +--- +kind: Service +apiVersion: v1 +metadata: + name: jenkins-ui + namespace: jenkins +spec: + type: NodePort + selector: + app: master + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + name: ui +--- +kind: Service +apiVersion: v1 +metadata: + name: jenkins-discovery + namespace: jenkins +spec: + selector: + app: master + ports: + - protocol: TCP + port: 50000 + targetPort: 50000 + name: agent +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: jenkins + namespace: jenkins +spec: +# tls: +# - secretName: tls + backend: + serviceName: jenkins-ui + servicePort: 8080 +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: agent-debian-testing-clang8 + namespace: jenkins +spec: + replicas: 1 + template: + metadata: + labels: + app: agent-debian-testing-clang8 + spec: + containers: + - name: agent-debian-testing-clang8 + image: gcr.io/llvm-windows-development/agent-debian-testing-clang8 + ports: + - containerPort: 22 + resources: + limits: + cpu: 30 + memory: 100Gi + requests: + cpu: 30 + memory: 100Gi +--- +kind: Service +apiVersion: v1 +metadata: + name: agent-debian-testing-clang8-ssh + namespace: jenkins +spec: + selector: + app: agent-debian-testing-clang8 + ports: + - protocol: TCP + port: 22 + name: ssh + type: NodePort diff --git a/local_setup.sh b/local_setup.sh new file mode 100755 index 0000000..8d2a4a9 --- /dev/null +++ b/local_setup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Run this script once to install all required tools and configure your machine. + + +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# install all the required tools for managing the cluster +sudo apt install -y google-cloud-sdk kubectl docker + +# configure gCloud +source "${ROOT_DIR}/k8s_config" +gcloud config set project ${GCP_PROJECT} +gcloud config set compute/zone ${GCP_ZONE} + +# setup docker for pushing containers +gcloud auth configure-docker \ No newline at end of file