Simple service to integrate harbormaster w/ buildkite

Harbormaster can send a request in url encoded form while buildkite
expects a POST request with json. Instead of modifying harbormastar or
buildkite this adds a simple proxy that accepts url encoded form and
creates a request that buildkite expects. To avoid potential abuse,
nginx asks for simple http auth credentials stored in harbormaster.

All build parameters passed by proxy are put into build metadata and as
ph_* env variables available during the build.

Secrets involved:
- harbormastert knows http-auth to proxy (stored in privatly and in k8
  buildkite/http-auth as auth file);
- proxy knows buildkite api token (mine atm);
- build agent knows conduit API token (mine atm),
  and SSH key of llvm-premerge-tests-bot (in k8 buildkite/github-ssh).

Sample build: https://reviews.llvm.org/harbormaster/build/64828/8/
https://buildkite.com/llvm-project/premerge/builds/48

containers/buildkite-premerge-debian/Dockerfile

@@ -10,4 +10,5 @@ RUN echo 'install buildkite' ;\
 
 COPY *.sh /usr/local/bin/
 RUN chmod og+rx /usr/local/bin/*.sh
+ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
 CMD ["start_agent.sh"]

containers/buildkite-premerge-debian/start_agent.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2020 Google LLC
 #
 # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
@@ -17,22 +17,18 @@
 USER=buildkite-agent
 SSD_ROOT="/mnt/disks/ssd0"
 AGENT_ROOT="${SSD_ROOT}/agent"
-CCACHE_PATH="${SSD_ROOT}/ccache"
 
-# prepare root folder for Jenkins agent
+# prepare work directory
 mkdir -p "${AGENT_ROOT}"
 chown -R ${USER}:${USER} "${AGENT_ROOT}"
-# TODO: this is needed if we want to use SSH auth.
-#mkdir -p /var/lib/buildkite-agent/.ssh
-#cp /mnt/ssh/id_rsa /var/lib/buildkite-agent/.ssh
-#cp /mnt/ssh/id_rsa.pub /var/lib/buildkite-agent/.ssh
-#chown -R ${USER}:${USER} /var/lib/buildkite-agent/.ssh
-
-# prepare folder for ccache
 mkdir -p "${CCACHE_PATH}"
 chown -R ${USER}:${USER} "${CCACHE_PATH}"
 
-# TODO(kuhnel): wipe the disk(s) on startup
+# /mnt/ssh should contain known_hosts, id_rsa and id_rsa.pub .
+mkdir -p /var/lib/buildkite-agent/.ssh
+cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh
+chmod 700 /var/lib/buildkite-agent/.ssh
+chmod 600 /var/lib/buildkite-agent/.ssh/*
+chown -R $USER:$USER /var/lib/buildkite-agent/.ssh
 
-# start the buildkite agent
 su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent"

kubernetes/buildkite/agents_premerge.yaml

@@ -37,6 +37,8 @@ spec:
         volumeMounts:
         - name: ssd
           mountPath: /mnt/disks/ssd0
+        - name: github-ssh
+          mountPath: /mnt/ssh
         env:
         - name: BUILDKITE_AGENT_TOKEN
           valueFrom:
@@ -45,11 +47,19 @@ spec:
               key: token
         - name: BUILDKITE_AGENT_TAGS
           value: "queue=premerge,os=linux"
+        - name: CONDUIT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: conduit-api-token
+              key: token
       volumes:
       - name: ssd
         hostPath:
           # directory location on host
           path: /mnt/disks/ssd0
           type: Directory
+      - name: github-ssh
+        secret:
+          secretName: github-ssh
       nodeSelector:
         cloud.google.com/gke-nodepool: jenkins-agents

kubernetes/phabricator-proxy/Deployment.yaml

@@ -0,0 +1,57 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: phabricator-proxy
+  namespace: buildkite
+spec:
+  selector:
+    matchLabels:
+      app: phabricator-proxy
+  replicas: 1 
+  template:
+    metadata:
+      labels:
+        app: phabricator-proxy
+    spec:
+      containers:
+      - name: phabricator-proxy
+        image: gcr.io/llvm-premerge-checks/phabricator-proxy
+        ports:
+        - containerPort: 8080
+        env:
+        - name: BUILDKITE_API_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: buildkite-api-token
+              key: token
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8080
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 2
+          failureThreshold: 5
+        resources:
+          limits:
+            cpu: 500m
+            memory: 1500Mi
+          requests:
+            cpu: 500m
+            memory: 1500Mi
+      nodeSelector:
+        cloud.google.com/gke-nodepool: default-pool

kubernetes/phabricator-proxy/Ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: nginx-ingress-build
+  namespace: buildkite
+  annotations:
+    kubernetes.io/ingress.global-static-ip-name: "web-static-ip"
+    kubernetes.io/ingress.class: "nginx"
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/auth-type: basic
+    nginx.ingress.kubernetes.io/auth-secret: http-auth
+    nginx.ingress.kubernetes.io/auth-realm: "LLVM pre-merge checks"
+spec:
+  tls:
+  - secretName: build-prod-tls
+    hosts:
+    - build.llvm-merge-guard.org
+  rules:
+  - host: build.llvm-merge-guard.org
+    http:
+      paths:
+      - backend:
+          serviceName: phabricator-proxy
+          servicePort: 8080

kubernetes/phabricator-proxy/Services.yaml

@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
 # Copyright 2020 Google LLC
 #
 # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
@@ -13,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-cat << EOF
+kind: Service
-steps:
+apiVersion: v1
-  - label: "bootstrap"
+metadata:
-    commands:
+  name: phabricator-proxy
-    - "git clone --depth 1 --branch \"${PREMERGE_SCRIPTS_BRANCH}\" https://github.com/google/llvm-premerge-checks.git"
+  namespace: buildkite
-    - "llvm-premerge-checks/scripts/buildkite/create_pipeline.py | tee /dev/tty | buildkite-agent pipeline upload"
+spec:
-    agents:
+  selector:
-        queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    app: phabricator-proxy
-        os: "linux"
+  ports:
-EOF
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080

kubernetes/phabricator-proxy/kustomization.yaml

@@ -0,0 +1,22 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: jenkins
+resources:
+  - Deployment.yaml
+  - Services.yaml
+  - Ingress.yaml
+

kubernetes/store_ssh_key.sh

@@ -33,4 +33,4 @@ fi
 
 kubectl create secret generic github-ssh-key --namespace jenkins \
   --from-file "$LOCAL_SSH_DIR/id_rsa" \
-    --from-file "$LOCAL_SSH_DIR/id_rsa.pub"
+  --from-file "$LOCAL_SSH_DIR/id_rsa.pub"

phabricator-proxy/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3
+
+RUN pip install flask gunicorn requests
+
+ADD main.py /
+
+CMD ["gunicorn", "--bind", "0.0.0.0:8080", "main:app"]

phabricator-proxy/README.md

@@ -0,0 +1,4 @@
+This is a small service to integrate Harbormaster and buildkite.
+
+Located at http://build.llvm-merge-guard.org behind http auth and is not
+publicly accessible as it's only used from Harbormaster. 

phabricator-proxy/build_deploy.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$(dirname ${DIR})"
+
+# get config options
+
+IMAGE_NAME="phabricator-proxy"
+
+docker build -t ${IMAGE_NAME} .
+read -p "Push to registry? [yN]" -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+  source "${ROOT_DIR}/k8s_config"
+  QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}"
+  docker tag ${IMAGE_NAME} ${QUALIFIED_NAME}
+  docker push ${QUALIFIED_NAME}
+fi

phabricator-proxy/main.py

@@ -0,0 +1,58 @@
+import flask
+import requests
+import os
+from urllib.parse import urlparse, parse_qs
+import json
+
+app = flask.Flask(__name__)
+app.config["DEBUG"] = True # TODO: make production
+buildkite_api_token = os.getenv("BUILDKITE_API_TOKEN", "")
+
+
+@app.route('/', methods=['GET'])
+def home():
+    return "Hi LLVM!"
+
+
+@app.route('/build', methods=['POST', 'GET'])
+def build():
+    app.logger.info('request: %s %s', flask.request, flask.request.url)
+    app.logger.info('headers: %s', flask.request.headers)
+    if flask.request.method == 'POST':
+        app.logger.info('data: %s', flask.request.data)
+        app.logger.info('form: %s', flask.request.form)
+        url = urlparse(flask.request.url)
+        params = parse_qs(url.query)
+        metadata = {}
+        build_env = {}
+        for k, v in params.items():
+            if len(v) == 1:
+                metadata[k] = v[0]
+                build_env['ph_' + k] = v[0]
+            else:
+                metadata[k] = v
+        branch = 'master'
+        if 'scripts_branch' in metadata:
+            branch = metadata['scripts_branch']
+        build_request = {
+            'commit': 'HEAD',
+            'branch':  branch,
+            'meta_data': metadata,
+            'env': build_env,
+        }
+        app.logger.info('buildkite request: %s', build_request)
+        headers = {'Authorization': f'Bearer {buildkite_api_token}'}
+        response = requests.post(
+            'https://api.buildkite.com/v2/organizations/llvm-project'
+            '/pipelines/premerge/builds',
+            json=build_request,
+            headers=headers)
+        app.logger.info('buildkite response: %s %s', response.status_code, response.text)
+        rjs = json.loads(response.text)
+        return rjs['id']
+    else:
+        return "expected POST request"
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0:8080')

scripts/buildkite/apply_patch.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+scripts/phabtalk/apply_patch2.py $ph_buildable_diff \
+  --token $CONDUIT_TOKEN \
+  --url $PHABRICATOR_HOST \
+  --comment-file apply_patch.txt \
+  --push-branch

scripts/buildkite/create_pipeline.py

@@ -22,14 +22,8 @@ if __name__ == '__main__':
   steps:
     - label: "build"
       commands:
-      - "git clone --depth 1 --branch '{script_branch}' https://github.com/google/llvm-premerge-checks.git"
+      - "git clone git@github.com:llvm-premerge-tests/llvm-project.git"
-      - "llvm-premerge-checks/scripts/run_buildkite.sh"
+      - "scripts/buildkite/apply_patch.sh" 
-      agents:
-          queue: "{queue}"
-          os: "linux"
-    - label: "parallel step"
-      commands:
-      - "echo do nothing"
       agents:
           queue: "{queue}"
           os: "linux"