diff --git a/containers/buildkite-windows/cloudbuild.yaml b/containers/buildkite-windows/cloudbuild.yaml index 2e6b3bd..0d6b523 100644 --- a/containers/buildkite-windows/cloudbuild.yaml +++ b/containers/buildkite-windows/cloudbuild.yaml @@ -6,11 +6,13 @@ steps: - name: 'gcr.io/${PROJECT_ID}/windows-builder' args: ['--image','windows-cloud/global/images/windows-server-2019-dc-core-for-containers-v20230113', - '--network', 'vpc-network', - '--subnetwork', 'subnetwork', - '--region', 'europe-west3', - '--zone', 'europe-west3-c', - '--machineType', "n2-standard-16", - '--diskType', 'pd-ssd', - '--command', 'gcloud auth configure-docker --quiet && docker build -t gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest . && docker push gcr.io/${PROJECT_ID}/buildkite-premerge-windows:latest'] -timeout: 7200s \ No newline at end of file + # '--network', 'vpc-network', + # '--subnetwork', 'subnetwork', + # '--region', 'europe-west3', + # '--zone', 'europe-west3-c', + # '--machineType', "n2-standard-16", + # '--diskType', 'pd-ssd', + '--command', 'gcloud auth configure-docker --quiet && docker build -t us-central1-docker.pkg.dev/llvm-premerge-checks/docker/buildkite-windows:latest . && docker push us-central1-docker.pkg.dev/llvm-premerge-checks/docker/buildkite-windows:latest' + # '--command', 'echo hi' + ] +timeout: 7200s diff --git a/containers/github-runner/Dockerfile b/containers/github-linux/Dockerfile similarity index 98% rename from containers/github-runner/Dockerfile rename to containers/github-linux/Dockerfile index 41efef0..1cee7d3 100644 --- a/containers/github-runner/Dockerfile +++ b/containers/github-linux/Dockerfile @@ -77,8 +77,8 @@ RUN groupadd -g 121 runner \ && mkdir -p /_work \ && chown -R runner:runner /_work /actions-runner; -COPY entrypoint.sh / -RUN chmod +x /entrypoint.sh +COPY entrypoint.sh token.sh / +RUN chmod +x /entrypoint.sh /token.sh # try: USER runner instead of gosu ENTRYPOINT ["/entrypoint.sh"] diff --git a/containers/github-linux/cloudbuild.yaml b/containers/github-linux/cloudbuild.yaml new file mode 100644 index 0000000..b00e570 --- /dev/null +++ b/containers/github-linux/cloudbuild.yaml @@ -0,0 +1,5 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux', '.' ] +images: +- 'us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest' \ No newline at end of file diff --git a/containers/github-linux/entrypoint.sh b/containers/github-linux/entrypoint.sh new file mode 100755 index 0000000..97a3c4c --- /dev/null +++ b/containers/github-linux/entrypoint.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +# Copyright 2021 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ueo pipefail + +export PATH=${PATH}:/actions-runner + +USER=runner +WORKDIR=${WORKDIR:-/_work} + +export SCCACHE_DIR="${WORKDIR}/sccache" +mkdir -p "${SCCACHE_DIR}" +chown -R ${USER}:${USER} "${SCCACHE_DIR}" +chmod oug+rw "${SCCACHE_DIR}" +gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server' +sccache --show-stats + +# Configure github runner. TODO: move to a separate file. +# Based on https://github.com/myoung34/docker-github-actions-runner/blob/master/entrypoint.sh +# licensed under MIT https://github.com/myoung34/docker-github-actions-runner/blob/master/LICENSE +export -n ACCESS_TOKEN +RUNNER_SCOPE=${RUNNER_SCOPE:-repo} +RUNNER_SCOPE="${RUNNER_SCOPE,,}" # to lowercase +_GITHUB_HOST=${GITHUB_HOST:="github.com"} +case ${RUNNER_SCOPE} in + org*) + [[ -z ${ORG_NAME} ]] && ( echo "ORG_NAME required for org runners"; exit 1 ) + _SHORT_URL="https://${_GITHUB_HOST}/${ORG_NAME}" + RUNNER_SCOPE="org" + ;; + + ent*) + [[ -z ${ENTERPRISE_NAME} ]] && ( echo "ENTERPRISE_NAME required for enterprise runners"; exit 1 ) + _SHORT_URL="https://${_GITHUB_HOST}/enterprises/${ENTERPRISE_NAME}" + RUNNER_SCOPE="enterprise" + ;; + + *) + [[ -z ${REPO_URL} ]] && ( echo "REPO_URL required for repo runners"; exit 1 ) + _SHORT_URL=${REPO_URL} + RUNNER_SCOPE="repo" + ;; +esac +_RUNNER_NAME=${RUNNER_NAME:-${RUNNER_NAME_PREFIX:-github-runner}-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo '')} +_LABELS=${LABELS:-default} +echo "Configuring" +echo "runner URL" "${_SHORT_URL}" +echo "workdir ${WORKDIR}" +echo "access token" "${ACCESS_TOKEN}" +echo "labels ${_LABELS}" +echo "runner name" "${_RUNNER_NAME}" + +echo "Obtaining the token of the runner" +_TOKEN=$(ACCESS_TOKEN="${ACCESS_TOKEN}" bash /token.sh) +RUNNER_TOKEN=$(echo "${_TOKEN}" | jq -r .token) +echo "RUNNER_TOKEN ${RUNNER_TOKEN}" + +gosu runner ./config.sh \ + --url "${_SHORT_URL}" \ + --token "${RUNNER_TOKEN}" \ + --name "${_RUNNER_NAME}" \ + --work "${WORKDIR}" \ + --labels "${_LABELS}" \ + --unattended \ + --replace + +[[ ! -d "${WORKDIR}" ]] && mkdir "${WORKDIR}" + +# exec /usr/bin/tini -g -- $@ +gosu runner "$@" diff --git a/containers/github-linux/token.sh b/containers/github-linux/token.sh new file mode 100755 index 0000000..c8e661f --- /dev/null +++ b/containers/github-linux/token.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# https://github.com/myoung34/docker-github-actions-runner/blob/master/token.sh +# Licensed under MIT +# https://github.com/myoung34/docker-github-actions-runner/blob/master/LICENSE + +set -euo pipefail + +_GITHUB_HOST=${GITHUB_HOST:="github.com"} + +# If URL is not github.com then use the enterprise api endpoint +if [[ ${GITHUB_HOST} = "github.com" ]]; then + URI="https://api.${_GITHUB_HOST}" +else + URI="https://${_GITHUB_HOST}/api/v3" +fi + +API_VERSION=v3 +API_HEADER="Accept: application/vnd.github.${API_VERSION}+json" +AUTH_HEADER="Authorization: token ${ACCESS_TOKEN}" +CONTENT_LENGTH_HEADER="Content-Length: 0" + +case ${RUNNER_SCOPE} in + org*) + _FULL_URL="${URI}/orgs/${ORG_NAME}/actions/runners/registration-token" + ;; + + ent*) + _FULL_URL="${URI}/enterprises/${ENTERPRISE_NAME}/actions/runners/registration-token" + ;; + + *) + _PROTO="https://" + # shellcheck disable=SC2116 + _URL="$(echo "${REPO_URL/${_PROTO}/}")" + _PATH="$(echo "${_URL}" | grep / | cut -d/ -f2-)" + _ACCOUNT="$(echo "${_PATH}" | cut -d/ -f1)" + _REPO="$(echo "${_PATH}" | cut -d/ -f2)" + _FULL_URL="${URI}/repos/${_ACCOUNT}/${_REPO}/actions/runners/registration-token" + ;; +esac + +RUNNER_TOKEN="$(curl -XPOST -fsSL \ + -H "${CONTENT_LENGTH_HEADER}" \ + -H "${AUTH_HEADER}" \ + -H "${API_HEADER}" \ + "${_FULL_URL}" \ +| jq -r '.token')" + +echo "{\"token\": \"${RUNNER_TOKEN}\", \"full_url\": \"${_FULL_URL}\"}" diff --git a/containers/github-runner/entrypoint.sh b/containers/github-runner/entrypoint.sh deleted file mode 100755 index c3dd6c3..0000000 --- a/containers/github-runner/entrypoint.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eo pipefail - -export PATH=${PATH}:/actions-runner - -USER=runner -RUNNER_WORKDIR="/_work" -set -u - -export SCCACHE_DIR="${RUNNER_WORKDIR}/sccache" -mkdir -p "${SCCACHE_DIR}" -chown -R ${USER}:${USER} "${SCCACHE_DIR}" -chmod oug+rw "${SCCACHE_DIR}" -gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server' -sccache --show-stats -_RUNNER_NAME=${RUNNER_NAME:-${RUNNER_NAME_PREFIX:-github-runner}-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo '')} -echo "Configuring" -echo "runner URL" "${ACTION_RUNNER_URL}" -echo "runner token" "${ACTION_RUNNER_TOKEN}" -echo "runner name" "${_RUNNER_NAME}" -gosu runner ./config.sh \ - --url "${ACTION_RUNNER_URL}" \ - --token "${ACTION_RUNNER_TOKEN}" \ - --name "${_RUNNER_NAME}" \ - --work "${RUNNER_WORKDIR}" \ - --labels "${ACTION_RUNNER_LABEL}" \ - --unattended \ - --replace - -# exec /usr/bin/tini -g -- $@ -gosu runner "$@" diff --git a/kubernetes/github/linux-test.yaml b/kubernetes/github/linux-test.yaml new file mode 100644 index 0000000..4755d6a --- /dev/null +++ b/kubernetes/github/linux-test.yaml @@ -0,0 +1,72 @@ +# Copyright 2023 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: github-linux-test +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + type: RollingUpdate + selector: + matchLabels: + app: github-linux-test + template: + metadata: + labels: + app: github-linux-test + spec: + containers: + - name: runner + image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest + resources: + limits: + cpu: 31 + memory: 80Gi + requests: + cpu: 31 + memory: 80Gi + volumeMounts: + - name: workdir + mountPath: /work + env: + - name: WORKDIR + value: "/work" + - name: ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: github-register-agent-pat + key: token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: RUNNER_NAME + value: "$(POD_NAME)" + - name: RUNNER_SCOPE + value: "org" + - name: ORG_NAME + value: "metafloworg" + - name: LABELS + value: "linux" + volumes: + - name: workdir + emptyDir: {} + nodeSelector: + cloud.google.com/gke-nodepool: linux-agents-2 + terminationGracePeriodSeconds: 30