Merge pull request #182 from google/buildkite

Simple service to integrate harbormaster w/ buildkite

containers/buildkite-premerge-debian/Dockerfile

@@ -10,4 +10,5 @@ RUN echo 'install buildkite' ;\
 
 COPY *.sh /usr/local/bin/
 RUN chmod og+rx /usr/local/bin/*.sh
+ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
 CMD ["start_agent.sh"]

containers/buildkite-premerge-debian/start_agent.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2020 Google LLC
 #
 # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
@@ -17,22 +17,18 @@
 USER=buildkite-agent
 SSD_ROOT="/mnt/disks/ssd0"
 AGENT_ROOT="${SSD_ROOT}/agent"
-CCACHE_PATH="${SSD_ROOT}/ccache"
 
-# prepare root folder for Jenkins agent
+# prepare work directory
 mkdir -p "${AGENT_ROOT}"
 chown -R ${USER}:${USER} "${AGENT_ROOT}"
-# TODO: this is needed if we want to use SSH auth.
-#mkdir -p /var/lib/buildkite-agent/.ssh
-#cp /mnt/ssh/id_rsa /var/lib/buildkite-agent/.ssh
-#cp /mnt/ssh/id_rsa.pub /var/lib/buildkite-agent/.ssh
-#chown -R ${USER}:${USER} /var/lib/buildkite-agent/.ssh
-
-# prepare folder for ccache
 mkdir -p "${CCACHE_PATH}"
 chown -R ${USER}:${USER} "${CCACHE_PATH}"
 
-# TODO(kuhnel): wipe the disk(s) on startup
+# /mnt/ssh should contain known_hosts, id_rsa and id_rsa.pub .
+mkdir -p /var/lib/buildkite-agent/.ssh
+cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh
+chmod 700 /var/lib/buildkite-agent/.ssh
+chmod 600 /var/lib/buildkite-agent/.ssh/*
+chown -R $USER:$USER /var/lib/buildkite-agent/.ssh
 
-# start the buildkite agent
 su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent"

kubernetes/buildkite/agents_premerge.yaml

@@ -37,6 +37,8 @@ spec:
         volumeMounts:
         - name: ssd
           mountPath: /mnt/disks/ssd0
+        - name: github-ssh
+          mountPath: /mnt/ssh
         env:
         - name: BUILDKITE_AGENT_TOKEN
           valueFrom:
@@ -45,11 +47,19 @@ spec:
               key: token
         - name: BUILDKITE_AGENT_TAGS
           value: "queue=premerge,os=linux"
+        - name: CONDUIT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: conduit-api-token
+              key: token
       volumes:
       - name: ssd
         hostPath:
           # directory location on host
           path: /mnt/disks/ssd0
           type: Directory
+      - name: github-ssh
+        secret:
+          secretName: github-ssh
       nodeSelector:
         cloud.google.com/gke-nodepool: jenkins-agents

kubernetes/phabricator-proxy/Deployment.yaml

@@ -0,0 +1,57 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: phabricator-proxy
+  namespace: buildkite
+spec:
+  selector:
+    matchLabels:
+      app: phabricator-proxy
+  replicas: 1 
+  template:
+    metadata:
+      labels:
+        app: phabricator-proxy
+    spec:
+      containers:
+      - name: phabricator-proxy
+        image: gcr.io/llvm-premerge-checks/phabricator-proxy
+        ports:
+        - containerPort: 8080
+        env:
+        - name: BUILDKITE_API_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: buildkite-api-token
+              key: token
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8080
+          periodSeconds: 10
+          timeoutSeconds: 5
+          successThreshold: 2
+          failureThreshold: 5
+        resources:
+          limits:
+            cpu: 500m
+            memory: 1500Mi
+          requests:
+            cpu: 500m
+            memory: 1500Mi
+      nodeSelector:
+        cloud.google.com/gke-nodepool: default-pool

kubernetes/phabricator-proxy/Ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: nginx-ingress-build
+  namespace: buildkite
+  annotations:
+    kubernetes.io/ingress.global-static-ip-name: "web-static-ip"
+    kubernetes.io/ingress.class: "nginx"
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/auth-type: basic
+    nginx.ingress.kubernetes.io/auth-secret: http-auth
+    nginx.ingress.kubernetes.io/auth-realm: "LLVM pre-merge checks"
+spec:
+  tls:
+  - secretName: build-prod-tls
+    hosts:
+    - build.llvm-merge-guard.org
+  rules:
+  - host: build.llvm-merge-guard.org
+    http:
+      paths:
+      - backend:
+          serviceName: phabricator-proxy
+          servicePort: 8080

kubernetes/phabricator-proxy/Services.yaml

@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
 # Copyright 2020 Google LLC
 #
 # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
@@ -13,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-cat << EOF
+kind: Service
-steps:
+apiVersion: v1
-  - label: "bootstrap"
+metadata:
-    commands:
+  name: phabricator-proxy
-    - "git clone --depth 1 --branch \"${PREMERGE_SCRIPTS_BRANCH}\" https://github.com/google/llvm-premerge-checks.git"
+  namespace: buildkite
-    - "llvm-premerge-checks/scripts/buildkite/create_pipeline.py | tee /dev/tty | buildkite-agent pipeline upload"
+spec:
-    agents:
+  selector:
-        queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    app: phabricator-proxy
-        os: "linux"
+  ports:
-EOF
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080

kubernetes/phabricator-proxy/kustomization.yaml

@@ -0,0 +1,22 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: jenkins
+resources:
+  - Deployment.yaml
+  - Services.yaml
+  - Ingress.yaml
+

kubernetes/store_ssh_key.sh

@@ -33,4 +33,4 @@ fi
 
 kubectl create secret generic github-ssh-key --namespace jenkins \
   --from-file "$LOCAL_SSH_DIR/id_rsa" \
-    --from-file "$LOCAL_SSH_DIR/id_rsa.pub"
+  --from-file "$LOCAL_SSH_DIR/id_rsa.pub"

phabricator-proxy/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3
+
+RUN pip install flask gunicorn requests
+
+ADD main.py /
+
+CMD ["gunicorn", "--bind", "0.0.0.0:8080", "main:app"]

phabricator-proxy/README.md

@@ -0,0 +1,4 @@
+This is a small service to integrate Harbormaster and buildkite.
+
+Located at http://build.llvm-merge-guard.org behind http auth and is not
+publicly accessible as it's only used from Harbormaster. 

phabricator-proxy/build_deploy.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$(dirname ${DIR})"
+
+# get config options
+
+IMAGE_NAME="phabricator-proxy"
+
+docker build -t ${IMAGE_NAME} .
+read -p "Push to registry? [yN]" -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+  source "${ROOT_DIR}/k8s_config"
+  QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}"
+  docker tag ${IMAGE_NAME} ${QUALIFIED_NAME}
+  docker push ${QUALIFIED_NAME}
+fi

phabricator-proxy/main.py

@@ -0,0 +1,58 @@
+import flask
+import requests
+import os
+from urllib.parse import urlparse, parse_qs
+import json
+
+app = flask.Flask(__name__)
+app.config["DEBUG"] = True # TODO: make production
+buildkite_api_token = os.getenv("BUILDKITE_API_TOKEN", "")
+
+
+@app.route('/', methods=['GET'])
+def home():
+    return "Hi LLVM!"
+
+
+@app.route('/build', methods=['POST', 'GET'])
+def build():
+    app.logger.info('request: %s %s', flask.request, flask.request.url)
+    app.logger.info('headers: %s', flask.request.headers)
+    if flask.request.method == 'POST':
+        app.logger.info('data: %s', flask.request.data)
+        app.logger.info('form: %s', flask.request.form)
+        url = urlparse(flask.request.url)
+        params = parse_qs(url.query)
+        metadata = {}
+        build_env = {}
+        for k, v in params.items():
+            if len(v) == 1:
+                metadata[k] = v[0]
+                build_env['ph_' + k] = v[0]
+            else:
+                metadata[k] = v
+        branch = 'master'
+        if 'scripts_branch' in metadata:
+            branch = metadata['scripts_branch']
+        build_request = {
+            'commit': 'HEAD',
+            'branch':  branch,
+            'meta_data': metadata,
+            'env': build_env,
+        }
+        app.logger.info('buildkite request: %s', build_request)
+        headers = {'Authorization': f'Bearer {buildkite_api_token}'}
+        response = requests.post(
+            'https://api.buildkite.com/v2/organizations/llvm-project'
+            '/pipelines/premerge/builds',
+            json=build_request,
+            headers=headers)
+        app.logger.info('buildkite response: %s %s', response.status_code, response.text)
+        rjs = json.loads(response.text)
+        return rjs['id']
+    else:
+        return "expected POST request"
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0:8080')

scripts/buildkite/apply_patch.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+scripts/phabtalk/apply_patch2.py $ph_buildable_diff \
+  --token $CONDUIT_TOKEN \
+  --url $PHABRICATOR_HOST \
+  --comment-file apply_patch.txt \
+  --push-branch

scripts/buildkite/create_pipeline.py

@@ -22,14 +22,8 @@ if __name__ == '__main__':
   steps:
     - label: "build"
       commands:
-      - "git clone --depth 1 --branch '{script_branch}' https://github.com/google/llvm-premerge-checks.git"
+      - "git clone git@github.com:llvm-premerge-tests/llvm-project.git"
-      - "llvm-premerge-checks/scripts/run_buildkite.sh"
+      - "scripts/buildkite/apply_patch.sh" 
-      agents:
-          queue: "{queue}"
-          os: "linux"
-    - label: "parallel step"
-      commands:
-      - "echo do nothing"
       agents:
           queue: "{queue}"
           os: "linux"