diff --git a/containers/buildkite-premerge-debian/Dockerfile b/containers/buildkite-premerge-debian/Dockerfile index f735d84..c6f3957 100644 --- a/containers/buildkite-premerge-debian/Dockerfile +++ b/containers/buildkite-premerge-debian/Dockerfile @@ -10,4 +10,5 @@ RUN echo 'install buildkite' ;\ COPY *.sh /usr/local/bin/ RUN chmod og+rx /usr/local/bin/*.sh +ENV CCACHE_PATH=/mnt/disks/ssd0/ccache CMD ["start_agent.sh"] \ No newline at end of file diff --git a/containers/buildkite-premerge-debian/start_agent.sh b/containers/buildkite-premerge-debian/start_agent.sh index d5487f5..e85e877 100755 --- a/containers/buildkite-premerge-debian/start_agent.sh +++ b/containers/buildkite-premerge-debian/start_agent.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2020 Google LLC # # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); @@ -17,22 +17,18 @@ USER=buildkite-agent SSD_ROOT="/mnt/disks/ssd0" AGENT_ROOT="${SSD_ROOT}/agent" -CCACHE_PATH="${SSD_ROOT}/ccache" -# prepare root folder for Jenkins agent +# prepare work directory mkdir -p "${AGENT_ROOT}" chown -R ${USER}:${USER} "${AGENT_ROOT}" -# TODO: this is needed if we want to use SSH auth. -#mkdir -p /var/lib/buildkite-agent/.ssh -#cp /mnt/ssh/id_rsa /var/lib/buildkite-agent/.ssh -#cp /mnt/ssh/id_rsa.pub /var/lib/buildkite-agent/.ssh -#chown -R ${USER}:${USER} /var/lib/buildkite-agent/.ssh - -# prepare folder for ccache mkdir -p "${CCACHE_PATH}" chown -R ${USER}:${USER} "${CCACHE_PATH}" -# TODO(kuhnel): wipe the disk(s) on startup +# /mnt/ssh should contain known_hosts, id_rsa and id_rsa.pub . +mkdir -p /var/lib/buildkite-agent/.ssh +cp /mnt/ssh/* /var/lib/buildkite-agent/.ssh +chmod 700 /var/lib/buildkite-agent/.ssh +chmod 600 /var/lib/buildkite-agent/.ssh/* +chown -R $USER:$USER /var/lib/buildkite-agent/.ssh -# start the buildkite agent su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent" \ No newline at end of file diff --git a/kubernetes/buildkite/agents_premerge.yaml b/kubernetes/buildkite/agents_premerge.yaml index d67d6d0..6a9dd94 100644 --- a/kubernetes/buildkite/agents_premerge.yaml +++ b/kubernetes/buildkite/agents_premerge.yaml @@ -37,6 +37,8 @@ spec: volumeMounts: - name: ssd mountPath: /mnt/disks/ssd0 + - name: github-ssh + mountPath: /mnt/ssh env: - name: BUILDKITE_AGENT_TOKEN valueFrom: @@ -45,11 +47,19 @@ spec: key: token - name: BUILDKITE_AGENT_TAGS value: "queue=premerge,os=linux" + - name: CONDUIT_TOKEN + valueFrom: + secretKeyRef: + name: conduit-api-token + key: token volumes: - name: ssd hostPath: # directory location on host path: /mnt/disks/ssd0 type: Directory + - name: github-ssh + secret: + secretName: github-ssh nodeSelector: cloud.google.com/gke-nodepool: jenkins-agents \ No newline at end of file diff --git a/kubernetes/phabricator-proxy/Deployment.yaml b/kubernetes/phabricator-proxy/Deployment.yaml new file mode 100644 index 0000000..de3ab5a --- /dev/null +++ b/kubernetes/phabricator-proxy/Deployment.yaml @@ -0,0 +1,57 @@ +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: phabricator-proxy + namespace: buildkite +spec: + selector: + matchLabels: + app: phabricator-proxy + replicas: 1 + template: + metadata: + labels: + app: phabricator-proxy + spec: + containers: + - name: phabricator-proxy + image: gcr.io/llvm-premerge-checks/phabricator-proxy + ports: + - containerPort: 8080 + env: + - name: BUILDKITE_API_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-api-token + key: token + readinessProbe: + httpGet: + path: / + port: 8080 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 2 + failureThreshold: 5 + resources: + limits: + cpu: 500m + memory: 1500Mi + requests: + cpu: 500m + memory: 1500Mi + nodeSelector: + cloud.google.com/gke-nodepool: default-pool \ No newline at end of file diff --git a/kubernetes/phabricator-proxy/Ingress.yaml b/kubernetes/phabricator-proxy/Ingress.yaml new file mode 100644 index 0000000..26aece0 --- /dev/null +++ b/kubernetes/phabricator-proxy/Ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: nginx-ingress-build + namespace: buildkite + annotations: + kubernetes.io/ingress.global-static-ip-name: "web-static-ip" + kubernetes.io/ingress.class: "nginx" + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: http-auth + nginx.ingress.kubernetes.io/auth-realm: "LLVM pre-merge checks" +spec: + tls: + - secretName: build-prod-tls + hosts: + - build.llvm-merge-guard.org + rules: + - host: build.llvm-merge-guard.org + http: + paths: + - backend: + serviceName: phabricator-proxy + servicePort: 8080 \ No newline at end of file diff --git a/containers/buildkite-premerge-debian/bootstrap_build.sh b/kubernetes/phabricator-proxy/Services.yaml old mode 100755 new mode 100644 similarity index 59% rename from containers/buildkite-premerge-debian/bootstrap_build.sh rename to kubernetes/phabricator-proxy/Services.yaml index 0f7d50a..f4e7177 --- a/containers/buildkite-premerge-debian/bootstrap_build.sh +++ b/kubernetes/phabricator-proxy/Services.yaml @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Copyright 2020 Google LLC # # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); @@ -13,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -cat << EOF -steps: - - label: "bootstrap" - commands: - - "git clone --depth 1 --branch \"${PREMERGE_SCRIPTS_BRANCH}\" https://github.com/google/llvm-premerge-checks.git" - - "llvm-premerge-checks/scripts/buildkite/create_pipeline.py | tee /dev/tty | buildkite-agent pipeline upload" - agents: - queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" - os: "linux" -EOF \ No newline at end of file +kind: Service +apiVersion: v1 +metadata: + name: phabricator-proxy + namespace: buildkite +spec: + selector: + app: phabricator-proxy + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/kubernetes/phabricator-proxy/kustomization.yaml b/kubernetes/phabricator-proxy/kustomization.yaml new file mode 100644 index 0000000..66bec82 --- /dev/null +++ b/kubernetes/phabricator-proxy/kustomization.yaml @@ -0,0 +1,22 @@ +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: jenkins +resources: + - Deployment.yaml + - Services.yaml + - Ingress.yaml + diff --git a/kubernetes/store_ssh_key.sh b/kubernetes/store_ssh_key.sh index 90147e1..955086a 100755 --- a/kubernetes/store_ssh_key.sh +++ b/kubernetes/store_ssh_key.sh @@ -33,4 +33,4 @@ fi kubectl create secret generic github-ssh-key --namespace jenkins \ --from-file "$LOCAL_SSH_DIR/id_rsa" \ - --from-file "$LOCAL_SSH_DIR/id_rsa.pub" \ No newline at end of file + --from-file "$LOCAL_SSH_DIR/id_rsa.pub" \ No newline at end of file diff --git a/phabricator-proxy/Dockerfile b/phabricator-proxy/Dockerfile new file mode 100644 index 0000000..ea87af2 --- /dev/null +++ b/phabricator-proxy/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3 + +RUN pip install flask gunicorn requests + +ADD main.py / + +CMD ["gunicorn", "--bind", "0.0.0.0:8080", "main:app"] \ No newline at end of file diff --git a/phabricator-proxy/README.md b/phabricator-proxy/README.md new file mode 100644 index 0000000..21367ad --- /dev/null +++ b/phabricator-proxy/README.md @@ -0,0 +1,4 @@ +This is a small service to integrate Harbormaster and buildkite. + +Located at http://build.llvm-merge-guard.org behind http auth and is not +publicly accessible as it's only used from Harbormaster. \ No newline at end of file diff --git a/phabricator-proxy/build_deploy.sh b/phabricator-proxy/build_deploy.sh new file mode 100755 index 0000000..8d8cd62 --- /dev/null +++ b/phabricator-proxy/build_deploy.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2019 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname ${DIR})" + +# get config options + +IMAGE_NAME="phabricator-proxy" + +docker build -t ${IMAGE_NAME} . +read -p "Push to registry? [yN]" -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]] +then + source "${ROOT_DIR}/k8s_config" + QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}" + docker tag ${IMAGE_NAME} ${QUALIFIED_NAME} + docker push ${QUALIFIED_NAME} +fi diff --git a/phabricator-proxy/main.py b/phabricator-proxy/main.py new file mode 100644 index 0000000..bb6285d --- /dev/null +++ b/phabricator-proxy/main.py @@ -0,0 +1,58 @@ +import flask +import requests +import os +from urllib.parse import urlparse, parse_qs +import json + +app = flask.Flask(__name__) +app.config["DEBUG"] = True # TODO: make production +buildkite_api_token = os.getenv("BUILDKITE_API_TOKEN", "") + + +@app.route('/', methods=['GET']) +def home(): + return "Hi LLVM!" + + +@app.route('/build', methods=['POST', 'GET']) +def build(): + app.logger.info('request: %s %s', flask.request, flask.request.url) + app.logger.info('headers: %s', flask.request.headers) + if flask.request.method == 'POST': + app.logger.info('data: %s', flask.request.data) + app.logger.info('form: %s', flask.request.form) + url = urlparse(flask.request.url) + params = parse_qs(url.query) + metadata = {} + build_env = {} + for k, v in params.items(): + if len(v) == 1: + metadata[k] = v[0] + build_env['ph_' + k] = v[0] + else: + metadata[k] = v + branch = 'master' + if 'scripts_branch' in metadata: + branch = metadata['scripts_branch'] + build_request = { + 'commit': 'HEAD', + 'branch': branch, + 'meta_data': metadata, + 'env': build_env, + } + app.logger.info('buildkite request: %s', build_request) + headers = {'Authorization': f'Bearer {buildkite_api_token}'} + response = requests.post( + 'https://api.buildkite.com/v2/organizations/llvm-project' + '/pipelines/premerge/builds', + json=build_request, + headers=headers) + app.logger.info('buildkite response: %s %s', response.status_code, response.text) + rjs = json.loads(response.text) + return rjs['id'] + else: + return "expected POST request" + + +if __name__ == '__main__': + app.run(host='0.0.0.0:8080') diff --git a/scripts/buildkite/apply_patch.sh b/scripts/buildkite/apply_patch.sh new file mode 100755 index 0000000..afdd8c2 --- /dev/null +++ b/scripts/buildkite/apply_patch.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +scripts/phabtalk/apply_patch2.py $ph_buildable_diff \ + --token $CONDUIT_TOKEN \ + --url $PHABRICATOR_HOST \ + --comment-file apply_patch.txt \ + --push-branch \ No newline at end of file diff --git a/scripts/buildkite/create_pipeline.py b/scripts/buildkite/create_pipeline.py index c900c64..f35d0ab 100755 --- a/scripts/buildkite/create_pipeline.py +++ b/scripts/buildkite/create_pipeline.py @@ -22,14 +22,8 @@ if __name__ == '__main__': steps: - label: "build" commands: - - "git clone --depth 1 --branch '{script_branch}' https://github.com/google/llvm-premerge-checks.git" - - "llvm-premerge-checks/scripts/run_buildkite.sh" - agents: - queue: "{queue}" - os: "linux" - - label: "parallel step" - commands: - - "echo do nothing" + - "git clone git@github.com:llvm-premerge-tests/llvm-project.git" + - "scripts/buildkite/apply_patch.sh" agents: queue: "{queue}" os: "linux"