github config

also trying to setup windows cloudbuild

containers/buildkite-windows/cloudbuild.yaml

@@ -6,11 +6,13 @@
 steps:
   - name: 'gcr.io/${PROJECT_ID}/windows-builder'
     args: ['--image','windows-cloud/global/images/windows-server-2019-dc-core-for-containers-v20230113',
-          '--network', 'vpc-network',
+          # '--network', 'vpc-network',
-          '--subnetwork', 'subnetwork',
+          # '--subnetwork', 'subnetwork',
-          '--region', 'europe-west3',
+          # '--region', 'europe-west3',
-          '--zone', 'europe-west3-c',
+          # '--zone', 'europe-west3-c',
-          '--machineType', "n2-standard-16",
+          # '--machineType', "n2-standard-16",
-          '--diskType', 'pd-ssd',
+          # '--diskType', 'pd-ssd',
-          '--command', 'gcloud auth configure-docker --quiet && docker build -t gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest . && docker push gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest']
+          '--command', 'gcloud auth configure-docker --quiet && docker build -t us-central1-docker.pkg.dev/llvm-premerge-checks/docker/buildkite-windows:latest . && docker push us-central1-docker.pkg.dev/llvm-premerge-checks/docker/buildkite-windows:latest'
-timeout: 7200s
+           # '--command', 'echo hi'
+          ]
+timeout: 7200s

containers/github-linux/Dockerfile

@@ -77,8 +77,8 @@ RUN groupadd -g 121 runner \
     && mkdir -p /_work \
     && chown -R runner:runner /_work /actions-runner;
 
-COPY entrypoint.sh /
+COPY entrypoint.sh token.sh /
-RUN chmod +x /entrypoint.sh
+RUN chmod +x /entrypoint.sh /token.sh
 # try: USER runner instead of gosu
 ENTRYPOINT ["/entrypoint.sh"]
 

containers/github-linux/cloudbuild.yaml

@@ -0,0 +1,5 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: [ 'build', '-t', 'us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux', '.' ]
+images:
+- 'us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest'

containers/github-linux/entrypoint.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ueo pipefail
+
+export PATH=${PATH}:/actions-runner
+
+USER=runner
+WORKDIR=${WORKDIR:-/_work}
+
+export SCCACHE_DIR="${WORKDIR}/sccache"
+mkdir -p "${SCCACHE_DIR}"
+chown -R ${USER}:${USER} "${SCCACHE_DIR}"
+chmod oug+rw "${SCCACHE_DIR}"
+gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server'
+sccache --show-stats
+
+# Configure github runner. TODO: move to a separate file.
+# Based on https://github.com/myoung34/docker-github-actions-runner/blob/master/entrypoint.sh
+# licensed under MIT https://github.com/myoung34/docker-github-actions-runner/blob/master/LICENSE
+export -n ACCESS_TOKEN
+RUNNER_SCOPE=${RUNNER_SCOPE:-repo}
+RUNNER_SCOPE="${RUNNER_SCOPE,,}" # to lowercase
+_GITHUB_HOST=${GITHUB_HOST:="github.com"}
+case ${RUNNER_SCOPE} in
+  org*)
+    [[ -z ${ORG_NAME} ]] && ( echo "ORG_NAME required for org runners"; exit 1 )
+    _SHORT_URL="https://${_GITHUB_HOST}/${ORG_NAME}"
+    RUNNER_SCOPE="org"
+    ;;
+
+  ent*)
+    [[ -z ${ENTERPRISE_NAME} ]] && ( echo "ENTERPRISE_NAME required for enterprise runners"; exit 1 )
+    _SHORT_URL="https://${_GITHUB_HOST}/enterprises/${ENTERPRISE_NAME}"
+    RUNNER_SCOPE="enterprise"
+    ;;
+
+  *)
+    [[ -z ${REPO_URL} ]] && ( echo "REPO_URL required for repo runners"; exit 1 )
+    _SHORT_URL=${REPO_URL}
+    RUNNER_SCOPE="repo"
+    ;;
+esac
+_RUNNER_NAME=${RUNNER_NAME:-${RUNNER_NAME_PREFIX:-github-runner}-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo '')}
+_LABELS=${LABELS:-default}
+echo "Configuring"
+echo "runner URL" "${_SHORT_URL}"
+echo "workdir ${WORKDIR}"
+echo "access token" "${ACCESS_TOKEN}"
+echo "labels ${_LABELS}"
+echo "runner name" "${_RUNNER_NAME}"
+
+echo "Obtaining the token of the runner"
+_TOKEN=$(ACCESS_TOKEN="${ACCESS_TOKEN}" bash /token.sh)
+RUNNER_TOKEN=$(echo "${_TOKEN}" | jq -r .token)
+echo "RUNNER_TOKEN ${RUNNER_TOKEN}"
+
+gosu runner ./config.sh \
+    --url "${_SHORT_URL}" \
+    --token "${RUNNER_TOKEN}" \
+    --name "${_RUNNER_NAME}" \
+    --work "${WORKDIR}" \
+    --labels "${_LABELS}" \
+    --unattended \
+    --replace
+
+[[ ! -d "${WORKDIR}" ]] && mkdir "${WORKDIR}"
+
+# exec /usr/bin/tini -g -- $@
+gosu runner "$@"

containers/github-linux/token.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# https://github.com/myoung34/docker-github-actions-runner/blob/master/token.sh
+# Licensed under MIT
+# https://github.com/myoung34/docker-github-actions-runner/blob/master/LICENSE
+
+set -euo pipefail
+
+_GITHUB_HOST=${GITHUB_HOST:="github.com"}
+
+# If URL is not github.com then use the enterprise api endpoint
+if [[ ${GITHUB_HOST} = "github.com" ]]; then
+  URI="https://api.${_GITHUB_HOST}"
+else
+  URI="https://${_GITHUB_HOST}/api/v3"
+fi
+
+API_VERSION=v3
+API_HEADER="Accept: application/vnd.github.${API_VERSION}+json"
+AUTH_HEADER="Authorization: token ${ACCESS_TOKEN}"
+CONTENT_LENGTH_HEADER="Content-Length: 0"
+
+case ${RUNNER_SCOPE} in
+  org*)
+    _FULL_URL="${URI}/orgs/${ORG_NAME}/actions/runners/registration-token"
+    ;;
+
+  ent*)
+    _FULL_URL="${URI}/enterprises/${ENTERPRISE_NAME}/actions/runners/registration-token"
+    ;;
+
+  *)
+    _PROTO="https://"
+    # shellcheck disable=SC2116
+    _URL="$(echo "${REPO_URL/${_PROTO}/}")"
+    _PATH="$(echo "${_URL}" | grep / | cut -d/ -f2-)"
+    _ACCOUNT="$(echo "${_PATH}" | cut -d/ -f1)"
+    _REPO="$(echo "${_PATH}" | cut -d/ -f2)"
+    _FULL_URL="${URI}/repos/${_ACCOUNT}/${_REPO}/actions/runners/registration-token"
+    ;;
+esac
+
+RUNNER_TOKEN="$(curl -XPOST -fsSL \
+  -H "${CONTENT_LENGTH_HEADER}" \
+  -H "${AUTH_HEADER}" \
+  -H "${API_HEADER}" \
+  "${_FULL_URL}" \
+| jq -r '.token')"
+
+echo "{\"token\": \"${RUNNER_TOKEN}\", \"full_url\": \"${_FULL_URL}\"}"

containers/github-runner/entrypoint.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2021 Google LLC
-#
-# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://llvm.org/LICENSE.txt
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -eo pipefail
-
-export PATH=${PATH}:/actions-runner
-
-USER=runner
-RUNNER_WORKDIR="/_work"
-set -u
-
-export SCCACHE_DIR="${RUNNER_WORKDIR}/sccache"
-mkdir -p "${SCCACHE_DIR}"
-chown -R ${USER}:${USER} "${SCCACHE_DIR}"
-chmod oug+rw "${SCCACHE_DIR}"
-gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server'
-sccache --show-stats
-_RUNNER_NAME=${RUNNER_NAME:-${RUNNER_NAME_PREFIX:-github-runner}-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo '')}
-echo "Configuring"
-echo "runner URL" "${ACTION_RUNNER_URL}"
-echo "runner token" "${ACTION_RUNNER_TOKEN}"
-echo "runner name" "${_RUNNER_NAME}"
-gosu runner ./config.sh \
-    --url "${ACTION_RUNNER_URL}" \
-    --token "${ACTION_RUNNER_TOKEN}" \
-    --name "${_RUNNER_NAME}" \
-    --work "${RUNNER_WORKDIR}" \
-    --labels "${ACTION_RUNNER_LABEL}" \
-    --unattended \
-    --replace
-
-# exec /usr/bin/tini -g -- $@
-gosu runner "$@"

kubernetes/github/linux-test.yaml

@@ -0,0 +1,72 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: github-linux-test
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: github-linux-test
+  template:
+    metadata:
+      labels:
+        app: github-linux-test
+    spec:
+      containers:
+        - name: runner
+          image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest
+          resources:
+            limits:
+              cpu: 31
+              memory: 80Gi
+            requests:
+              cpu: 31
+              memory: 80Gi
+          volumeMounts:
+            - name: workdir
+              mountPath: /work
+          env:
+            - name: WORKDIR
+              value: "/work"
+            - name: ACCESS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: github-register-agent-pat
+                  key: token
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                 fieldPath: metadata.name
+            - name: RUNNER_NAME
+              value: "$(POD_NAME)"
+            - name: RUNNER_SCOPE
+              value: "org"
+            - name: ORG_NAME
+              value: "metafloworg"
+            - name: LABELS
+              value: "linux"
+      volumes:
+        - name: workdir
+          emptyDir: {}
+      nodeSelector:
+        cloud.google.com/gke-nodepool: linux-agents-2
+      terminationGracePeriodSeconds: 30