1
0
Fork 0

action runners controller setup

This commit is contained in:
Mikhail Goncharov 2023-11-10 15:42:18 +01:00
parent d567c40f5a
commit 8d30467dde
5 changed files with 308 additions and 68 deletions

View file

@ -0,0 +1,29 @@
# Installation
Install helm (along with other tools in `local_setup.sh`).
Once per cluster:
`helm install arc --namespace "arc-system" --create-namespace oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller`
## Add new set of runners
Create runner set first time:
- copy 'values.yml` and updata parameters: APP ID, target node set, repo etc
- run
```
helm install <runner-set> --namespace <k8s-namespace> --create-namespace \
--values=$(readlink -f <values.yml>)
--set-file=githubConfigSecret.github_app_private_key=$(readlink -f <pem>) \
oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set
```
After that update values.yml to use `githubConfigSecret: arc-runner-set-gha-rs-github-secret`.
## Update
Example command for linux set:
`helm upgrade arc-google-linux --namespace arc-linux-prod -f $(readlink -f ~/src/merge-checks/actions-runner-controller/values-llvm.yaml) oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set`

View file

@ -0,0 +1,46 @@
# See full list settings doc in https://github.com/actions/actions-runner-controller/tree/master/charts/actions-runner-controller.
githubConfigUrl: "https://github.com/llvm/llvm-project"
# Created by first installation.
githubConfigSecret: arc-runner-set-gha-rs-github-secret
minRunners: 1
maxRunners: 3
runnerGroup: "generic-google-cloud"
## template for each runner Pod
template:
spec:
containers:
- name: runner
image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest
command: ["/bin/bash"]
args: ["-c", "/entrypoint.sh /home/runner/run.sh"]
env:
- name: ACTIONS_RUNNER_CONTAINER_HOOKS
value: /home/runner/k8s/index.js
- name: ACTIONS_RUNNER_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER
value: "false"
- name: WORKDIR
value: "/home/runner/_work"
resources:
limits:
cpu: 31
memory: 80Gi
requests:
cpu: 31
memory: 80Gi
volumeMounts:
- name: work
mountPath: /home/runner/_work
volumes:
- name: work
emptyDir: {}
nodeSelector:
cloud.google.com/gke-nodepool: linux-agents-2

View file

@ -0,0 +1,194 @@
# See options doc in https://github.com/actions/actions-runner-controller/tree/master/charts/actions-runner-controller
## githubConfigUrl is the GitHub url for where you want to configure runners
## ex: https://github.com/myorg/myrepo or https://github.com/myorg
githubConfigUrl: "https://github.com/metafloworg/llvm-project"
## githubConfigSecret is the k8s secrets to use when auth with GitHub API.
#githubConfigSecret:
### GitHub Apps Configuration
## NOTE: IDs MUST be strings, use quotes
#github_app_id: "427039"
#github_app_installation_id: "43840704"
## Pass --set-file=githubConfigSecret.github_app_private_key=<path to pem>
# First installation creates this secret.
githubConfigSecret: arc-runner-set-gha-rs-github-secret
## proxy can be used to define proxy settings that will be used by the
## controller, the listener and the runner of this scale set.
#
# proxy:
# http:
# url: http://proxy.com:1234
# credentialSecretRef: proxy-auth # a secret with `username` and `password` keys
# https:
# url: http://proxy.com:1234
# credentialSecretRef: proxy-auth # a secret with `username` and `password` keys
# noProxy:
# - example.com
# - example.org
## maxRunners is the max number of runners the autoscaling runner set will scale up to.
maxRunners: 3
## minRunners is the min number of runners the autoscaling runner set will scale down to.
minRunners: 1
# runnerGroup: "default"
## name of the runner scale set to create. Defaults to the helm release name
# runnerScaleSetName: ""
## A self-signed CA certificate for communication with the GitHub server can be
## provided using a config map key selector. If `runnerMountPath` is set, for
## each runner pod ARC will:
## - create a `github-server-tls-cert` volume containing the certificate
## specified in `certificateFrom`
## - mount that volume on path `runnerMountPath`/{certificate name}
## - set NODE_EXTRA_CA_CERTS environment variable to that same path
## - set RUNNER_UPDATE_CA_CERTS environment variable to "1" (as of version
## 2.303.0 this will instruct the runner to reload certificates on the host)
##
## If any of the above had already been set by the user in the runner pod
## template, ARC will observe those and not overwrite them.
## Example configuration:
#
# githubServerTLS:
# certificateFrom:
# configMapKeyRef:
# name: config-map-name
# key: ca.crt
# runnerMountPath: /usr/local/share/ca-certificates/
## Container mode is an object that provides out-of-box configuration
## for dind and kubernetes mode. Template will be modified as documented under the
## template object.
##
## If any customization is required for dind or kubernetes mode, containerMode should remain
## empty, and configuration should be applied to the template.
# containerMode:
# type: "dind" ## type can be set to dind or kubernetes
# ## the following is required when containerMode.type=kubernetes
# kubernetesModeWorkVolumeClaim:
# accessModes: ["ReadWriteOnce"]
# # For local testing, use https://github.com/openebs/dynamic-localpv-provisioner/blob/develop/docs/quickstart.md to provide dynamic provision volume with storageClassName: openebs-hostpath
# storageClassName: "dynamic-blob-storage"
# resources:
# requests:
# storage: 1Gi
# kubernetesModeServiceAccount:
# annotations:
## template is the PodSpec for each listener Pod
## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
# listenerTemplate:
# spec:
# containers:
# # Use this section to append additional configuration to the listener container.
# # If you change the name of the container, the configuration will not be applied to the listener,
# # and it will be treated as a side-car container.
# - name: listener
# securityContext:
# runAsUser: 1000
# # Use this section to add the configuration of a side-car container.
# # Comment it out or remove it if you don't need it.
# # Spec for this container will be applied as is without any modifications.
# - name: side-car
# image: example-sidecar
## template is the PodSpec for each runner Pod
## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
## template.spec will be modified if you change the container mode
## with containerMode.type=dind, we will populate the template.spec with following pod spec
## template:
## spec:
## initContainers:
## - name: init-dind-externals
## image: ghcr.io/actions/actions-runner:latest
## command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"]
## volumeMounts:
## - name: dind-externals
## mountPath: /home/runner/tmpDir
## containers:
## - name: runner
## image: ghcr.io/actions/actions-runner:latest
## command: ["/home/runner/run.sh"]
## env:
## - name: DOCKER_HOST
## value: unix:///run/docker/docker.sock
## volumeMounts:
## - name: work
## mountPath: /home/runner/_work
## - name: dind-sock
## mountPath: /run/docker
## readOnly: true
## - name: dind
## image: docker:dind
## args:
## - dockerd
## - --host=unix:///run/docker/docker.sock
## - --group=$(DOCKER_GROUP_GID)
## env:
## - name: DOCKER_GROUP_GID
## value: "123"
## securityContext:
## privileged: true
## volumeMounts:
## - name: work
## mountPath: /home/runner/_work
## - name: dind-sock
## mountPath: /run/docker
## - name: dind-externals
## mountPath: /home/runner/externals
## volumes:
## - name: work
## emptyDir: {}
## - name: dind-sock
## emptyDir: {}
## - name: dind-externals
## emptyDir: {}
######################################################################################################
## with containerMode.type=kubernetes, we will populate the template.spec with following pod spec
template:
spec:
containers:
- name: runner
image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest
command: ["/bin/bash"]
args: ["-c", "/entrypoint.sh /home/runner/run.sh"]
env:
- name: ACTIONS_RUNNER_CONTAINER_HOOKS
value: /home/runner/k8s/index.js
- name: ACTIONS_RUNNER_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER
value: "false"
- name: WORKDIR
value: "/home/runner/_work"
resources:
limits:
cpu: 31
memory: 80Gi
requests:
cpu: 31
memory: 80Gi
volumeMounts:
- name: work
mountPath: /home/runner/_work
volumes:
- name: work
emptyDir: {}
nodeSelector:
cloud.google.com/gke-nodepool: linux-agents-2
## Optional controller service account that needs to have required Role and RoleBinding
## to operate this gha-runner-scale-set installation.
## The helm chart will try to find the controller deployment and its service account at installation time.
## In case the helm chart can't find the right service account, you can explicitly pass in the following value
## to help it finish RoleBinding with the right service account.
## Note: if your controller is installed to only watch a single namespace, you have to pass these values explicitly.
# controllerServiceAccount:
# namespace: arc-system
# name: test-arc-gha-runner-scale-set-controller

View file

@ -6,7 +6,7 @@ RUN echo 'intall packages'; \
apt-get install -y --no-install-recommends \ apt-get install -y --no-install-recommends \
gosu \ gosu \
locales openssh-client gnupg ca-certificates \ locales openssh-client gnupg ca-certificates \
zip wget curl git \ zip unzip wget curl git \
gdb build-essential \ gdb build-essential \
ninja-build \ ninja-build \
libelf-dev libffi-dev gcc-multilib libmpfr-dev libpfm4-dev \ libelf-dev libffi-dev gcc-multilib libmpfr-dev libpfm4-dev \
@ -65,21 +65,41 @@ RUN curl -o sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz -L https://github.co
&& ls -la /usr/bin/sccache \ && ls -la /usr/bin/sccache \
&& sccache --version && sccache --version
WORKDIR /actions-runner
RUN cd /actions-runner \
&& curl -o actions-runner-linux-x64-2.308.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.308.0/actions-runner-linux-x64-2.308.0.tar.gz \
&& echo "9f994158d49c5af39f57a65bf1438cbae4968aec1e4fec132dd7992ad57c74fa actions-runner-linux-x64-2.308.0.tar.gz" | shasum -a 256 -c \
&& tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz
RUN groupadd -g 121 runner \ RUN groupadd -g 121 runner \
&& useradd -mr -d /home/runner -u 1001 -g 121 runner \ && useradd -mr -d /home/runner -u 1001 -g 121 runner \
&& mkdir -p /_work \ && mkdir -p /home/runner/_work
&& chown -R runner:runner /_work /actions-runner;
COPY entrypoint.sh token.sh / WORKDIR /home/runner
RUN chmod +x /entrypoint.sh /token.sh
# try: USER runner instead of gosu
ENTRYPOINT ["/entrypoint.sh"]
CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] # https://github.com/actions/runner/releases
ARG RUNNER_VERSION="2.311.0"
ARG RUNNER_ARCH="x64"
RUN curl -f -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz \
&& tar xzf ./runner.tar.gz \
&& rm runner.tar.gz
# https://github.com/actions/runner-container-hooks/releases
ARG RUNNER_CONTAINER_HOOKS_VERSION="0.4.0"
RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-container-hooks/releases/download/v${RUNNER_CONTAINER_HOOKS_VERSION}/actions-runner-hooks-k8s-${RUNNER_CONTAINER_HOOKS_VERSION}.zip \
&& unzip ./runner-container-hooks.zip -d ./k8s \
&& rm runner-container-hooks.zip
RUN chown -R runner:runner /home/runner
# Copy entrypoint but script. There is no point of adding
# `ENTRYPOINT ["/entrypoint.sh"]` as k8s runs it with explicit command and
# entrypoint is ignored.
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
# USER runner
# We don't need entry point and token logic as it's all orchestrated by action runner controller.
# BUT we need to start server etc
# RUN chmod +x /entrypoint.sh /token.sh
# # try: USER runner instead of gosu
# ENTRYPOINT ["/entrypoint.sh"]
# CMD ["sleep", "infinity"]
# CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"]

View file

@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Copyright 2021 Google LLC # Copyright 2023 Google LLC
# #
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -15,10 +15,10 @@
# limitations under the License. # limitations under the License.
set -ueo pipefail set -ueo pipefail
export PATH=${PATH}:/actions-runner export PATH=${PATH}:/home/runner
USER=runner USER=runner
WORKDIR=${WORKDIR:-/_work} WORKDIR=${WORKDIR:-/home/runner/_work}
export SCCACHE_DIR="${WORKDIR}/sccache" export SCCACHE_DIR="${WORKDIR}/sccache"
mkdir -p "${SCCACHE_DIR}" mkdir -p "${SCCACHE_DIR}"
@ -27,56 +27,7 @@ chmod oug+rw "${SCCACHE_DIR}"
gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server' gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server'
sccache --show-stats sccache --show-stats
# Configure github runner. TODO: move to a separate file. [[ ! -d "${WORKDIR}" ]] && mkdir -p "${WORKDIR}"
# Based on https://github.com/myoung34/docker-github-actions-runner/blob/master/entrypoint.sh
# licensed under MIT https://github.com/myoung34/docker-github-actions-runner/blob/master/LICENSE
export -n ACCESS_TOKEN
RUNNER_SCOPE=${RUNNER_SCOPE:-repo}
RUNNER_SCOPE="${RUNNER_SCOPE,,}" # to lowercase
_GITHUB_HOST=${GITHUB_HOST:="github.com"}
case ${RUNNER_SCOPE} in
org*)
[[ -z ${ORG_NAME} ]] && ( echo "ORG_NAME required for org runners"; exit 1 )
_SHORT_URL="https://${_GITHUB_HOST}/${ORG_NAME}"
RUNNER_SCOPE="org"
;;
ent*)
[[ -z ${ENTERPRISE_NAME} ]] && ( echo "ENTERPRISE_NAME required for enterprise runners"; exit 1 )
_SHORT_URL="https://${_GITHUB_HOST}/enterprises/${ENTERPRISE_NAME}"
RUNNER_SCOPE="enterprise"
;;
*)
[[ -z ${REPO_URL} ]] && ( echo "REPO_URL required for repo runners"; exit 1 )
_SHORT_URL=${REPO_URL}
RUNNER_SCOPE="repo"
;;
esac
_RUNNER_NAME=${RUNNER_NAME:-${RUNNER_NAME_PREFIX:-github-runner}-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo '')}
_LABELS=${LABELS:-default}
echo "Configuring"
echo "runner URL" "${_SHORT_URL}"
echo "workdir ${WORKDIR}"
echo "access token" "${ACCESS_TOKEN}"
echo "labels ${_LABELS}"
echo "runner name" "${_RUNNER_NAME}"
echo "Obtaining the token of the runner"
_TOKEN=$(ACCESS_TOKEN="${ACCESS_TOKEN}" bash /token.sh)
RUNNER_TOKEN=$(echo "${_TOKEN}" | jq -r .token)
echo "RUNNER_TOKEN ${RUNNER_TOKEN}"
gosu runner ./config.sh \
--url "${_SHORT_URL}" \
--token "${RUNNER_TOKEN}" \
--name "${_RUNNER_NAME}" \
--work "${WORKDIR}" \
--labels "${_LABELS}" \
--unattended \
--replace
[[ ! -d "${WORKDIR}" ]] && mkdir "${WORKDIR}"
# exec /usr/bin/tini -g -- $@ # exec /usr/bin/tini -g -- $@
gosu runner "$@" gosu runner "$@"