Forked docker image for premerge

- added simple script to define pipeline, it will check out from this
  repo later

- refactored docker container, e.g. merged most of the RUN steps into
  one and moved steps that are likely to change closer to the end. That
  should improve rebuild / upload speed.

Sample run: https://buildkite.com/llvm-project/premerge/builds/14

containers/base-debian/Dockerfile

@@ -0,0 +1,30 @@
+FROM debian:testing
+
+RUN echo 'intall build dependencies'; \
+    echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list ;\
+    apt-get update ;\
+    apt-get install -y --no-install-recommends \
+        locales openssh-client\
+        cmake ninja-build git ca-certificates clang lld ccache python3 build-essential gdb \
+        clang-tidy clang-format \
+        python3-psutil zip wget \
+        openjdk-11-jdk \
+        python3-pip python3-setuptools \
+        swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \
+    apt-get clean; \
+    echo 'configure locale'; \
+    sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
+    locale-gen ;\
+    echo 'make python 3 default'; \
+    rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python; \
+    pip3 install wheel
+
+# Configure locale
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+# Install python dependencies for the scripts. ADD will check contentents of a file for changes changed.
+# TODO: that should be done during the build as it will pull this repo anyway and will have latest version.
+ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt" requirements.txt
+RUN pip3 install -r requirements.txt

containers/build_deploy.sh

@@ -19,12 +19,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="$(dirname ${DIR})"
 
 # get config options
-source "${ROOT_DIR}/k8s_config"
 
 IMAGE_NAME="${1%/}"
-QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}"
 
 cd "${DIR}/${IMAGE_NAME}"
 docker build -t ${IMAGE_NAME} .
-docker tag ${IMAGE_NAME} ${QUALIFIED_NAME}
+read -p "Push to registry? [yN]" -n 1 -r
-docker push ${QUALIFIED_NAME}
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+  source "${ROOT_DIR}/k8s_config"
+  QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}"
+  docker tag ${IMAGE_NAME} ${QUALIFIED_NAME}
+  docker push ${QUALIFIED_NAME}
+fi

containers/buildkite-premerge-debian/Dockerfile

@@ -0,0 +1,12 @@
+FROM gcr.io/llvm-premerge-checks/base-debian:latest
+
+RUN echo 'install buildkite' ;\
+    apt-get install -y apt-transport-https gnupg;\
+    sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\
+    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 ;\
+    apt-get update ;\
+    apt-get install -y buildkite-agent; \
+    apt-get clean;
+
+COPY *.sh /usr/local/bin/
+CMD ["start_agent.sh"]

containers/buildkite-premerge-debian/bootstrap_build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cat << EOF
+steps:
+  - label: ":sparkles: success"
+    command: echo "bootstrap success"
+    agents:
+        queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+        os: "linux"
+EOF

containers/buildkite-premerge-debian/start_agent.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Buildkite installation creates 'buildkite-agent' user.
+USER=buildkite-agent
+SSD_ROOT="/mnt/disks/ssd0"
+AGENT_ROOT="${SSD_ROOT}/agent"
+CCACHE_PATH="${SSD_ROOT}/ccache"
+
+# prepare root folder for Jenkins agent
+mkdir -p "${AGENT_ROOT}"
+chown -R ${USER}:${USER} "${AGENT_ROOT}"
+# TODO: this is needed if we want to use SSH auth.
+#mkdir -p /var/lib/buildkite-agent/.ssh
+#cp /mnt/ssh/id_rsa /var/lib/buildkite-agent/.ssh
+#cp /mnt/ssh/id_rsa.pub /var/lib/buildkite-agent/.ssh
+#chown -R ${USER}:${USER} /var/lib/buildkite-agent/.ssh
+
+# prepare folder for ccache
+mkdir -p "${CCACHE_PATH}"
+chown -R ${USER}:${USER} "${CCACHE_PATH}"
+
+# TODO(kuhnel): wipe the disk(s) on startup
+
+# start the buildkite agent
+su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent"

kubernetes/buildkite/agents_premerge.yaml

@@ -15,7 +15,7 @@
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: agent-premerge-debian
+  name: premerge-debian
   namespace: buildkite
 spec:
   replicas: 1
@@ -25,8 +25,8 @@ spec:
         app: agent-premerge-debian
     spec:
       containers:
-      - name: agent-debian-testing-buildkite
+      - name: buildkite-premerge-debian
-        image: gcr.io/llvm-premerge-checks/agent-debian-testing-buildkite
+        image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian
         resources:
           limits:
             cpu: 30