action runners controller setup

2023-11-10 15:42:18 +01:00 · 2023-11-10 15:42:18 +01:00 · 8d30467dde
commit 8d30467dde
parent d567c40f5a
5 changed files with 308 additions and 68 deletions
--- a/actions-runner-controller/README.md
+++ b/actions-runner-controller/README.md
@ -0,0 +1,29 @@
+# Installation
+
+Install helm (along with other tools in `local_setup.sh`).
+
+Once per cluster:
+
+`helm install arc  --namespace "arc-system" --create-namespace oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller`
+
+## Add new set of runners
+
+Create runner set first time:
+
+- copy 'values.yml` and updata parameters: APP ID, target node set, repo etc
+- run
+
+```
+helm install <runner-set> --namespace <k8s-namespace> --create-namespace \
+  --values=$(readlink -f <values.yml>)
+  --set-file=githubConfigSecret.github_app_private_key=$(readlink -f <pem>) \
+  oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set
+```
+
+After that update values.yml to use `githubConfigSecret: arc-runner-set-gha-rs-github-secret`.
+
+## Update
+
+Example command for linux set:
+
+`helm upgrade arc-google-linux --namespace arc-linux-prod -f $(readlink -f ~/src/merge-checks/actions-runner-controller/values-llvm.yaml) oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set`
--- a/actions-runner-controller/values-llvm.yaml
+++ b/actions-runner-controller/values-llvm.yaml
@ -0,0 +1,46 @@
+# See full list settings doc in https://github.com/actions/actions-runner-controller/tree/master/charts/actions-runner-controller.
+
+githubConfigUrl: "https://github.com/llvm/llvm-project"
+
+# Created by first installation.
+githubConfigSecret: arc-runner-set-gha-rs-github-secret
+
+minRunners: 1
+maxRunners: 3
+
+runnerGroup: "generic-google-cloud"
+
+## template for each runner Pod
+template:
+  spec:
+    containers:
+    - name: runner
+      image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest
+      command: ["/bin/bash"]
+      args: ["-c", "/entrypoint.sh /home/runner/run.sh"]
+      env:
+        - name: ACTIONS_RUNNER_CONTAINER_HOOKS
+          value: /home/runner/k8s/index.js
+        - name: ACTIONS_RUNNER_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER
+          value: "false"
+        - name: WORKDIR
+          value: "/home/runner/_work"
+      resources:
+        limits:
+          cpu: 31
+          memory: 80Gi
+        requests:
+          cpu: 31
+          memory: 80Gi
+      volumeMounts:
+        - name: work
+          mountPath: /home/runner/_work
+    volumes:
+      - name: work
+        emptyDir: {}
+    nodeSelector:
+        cloud.google.com/gke-nodepool: linux-agents-2
--- a/actions-runner-controller/values-test.yaml
+++ b/actions-runner-controller/values-test.yaml
@ -0,0 +1,194 @@
+# See options doc in https://github.com/actions/actions-runner-controller/tree/master/charts/actions-runner-controller
+
+## githubConfigUrl is the GitHub url for where you want to configure runners
+## ex: https://github.com/myorg/myrepo or https://github.com/myorg
+githubConfigUrl: "https://github.com/metafloworg/llvm-project"
+
+## githubConfigSecret is the k8s secrets to use when auth with GitHub API.
+#githubConfigSecret:
+  ### GitHub Apps Configuration
+  ## NOTE: IDs MUST be strings, use quotes
+  #github_app_id: "427039"
+  #github_app_installation_id: "43840704"
+  ## Pass --set-file=githubConfigSecret.github_app_private_key=<path to pem>
+# First installation creates this secret.
+githubConfigSecret: arc-runner-set-gha-rs-github-secret
+
+## proxy can be used to define proxy settings that will be used by the
+## controller, the listener and the runner of this scale set.
+#
+# proxy:
+#   http:
+#     url: http://proxy.com:1234
+#     credentialSecretRef: proxy-auth # a secret with `username` and `password` keys
+#   https:
+#     url: http://proxy.com:1234
+#     credentialSecretRef: proxy-auth # a secret with `username` and `password` keys
+#   noProxy:
+#     - example.com
+#     - example.org
+
+## maxRunners is the max number of runners the autoscaling runner set will scale up to.
+maxRunners: 3
+
+## minRunners is the min number of runners the autoscaling runner set will scale down to.
+minRunners: 1
+
+# runnerGroup: "default"
+
+## name of the runner scale set to create.  Defaults to the helm release name
+# runnerScaleSetName: ""
+
+## A self-signed CA certificate for communication with the GitHub server can be
+## provided using a config map key selector. If `runnerMountPath` is set, for
+## each runner pod ARC will:
+## - create a `github-server-tls-cert` volume containing the certificate
+##   specified in `certificateFrom`
+## - mount that volume on path `runnerMountPath`/{certificate name}
+## - set NODE_EXTRA_CA_CERTS environment variable to that same path
+## - set RUNNER_UPDATE_CA_CERTS environment variable to "1" (as of version
+##   2.303.0 this will instruct the runner to reload certificates on the host)
+##
+## If any of the above had already been set by the user in the runner pod
+## template, ARC will observe those and not overwrite them.
+## Example configuration:
+#
+# githubServerTLS:
+#   certificateFrom:
+#     configMapKeyRef:
+#       name: config-map-name
+#       key: ca.crt
+#   runnerMountPath: /usr/local/share/ca-certificates/
+
+## Container mode is an object that provides out-of-box configuration
+## for dind and kubernetes mode. Template will be modified as documented under the
+## template object.
+##
+## If any customization is required for dind or kubernetes mode, containerMode should remain
+## empty, and configuration should be applied to the template.
+# containerMode:
+#   type: "dind"  ## type can be set to dind or kubernetes
+#   ## the following is required when containerMode.type=kubernetes
+#   kubernetesModeWorkVolumeClaim:
+#     accessModes: ["ReadWriteOnce"]
+#     # For local testing, use https://github.com/openebs/dynamic-localpv-provisioner/blob/develop/docs/quickstart.md to provide dynamic provision volume with storageClassName: openebs-hostpath
+#     storageClassName: "dynamic-blob-storage"
+#     resources:
+#       requests:
+#         storage: 1Gi
+#   kubernetesModeServiceAccount:
+#     annotations:
+
+## template is the PodSpec for each listener Pod
+## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
+# listenerTemplate:
+#   spec:
+#     containers:
+#     # Use this section to append additional configuration to the listener container.
+#     # If you change the name of the container, the configuration will not be applied to the listener,
+#     # and it will be treated as a side-car container.
+#     - name: listener
+#       securityContext:
+#         runAsUser: 1000
+#     # Use this section to add the configuration of a side-car container.
+#     # Comment it out or remove it if you don't need it.
+#     # Spec for this container will be applied as is without any modifications.
+#     - name: side-car
+#       image: example-sidecar
+
+## template is the PodSpec for each runner Pod
+## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
+  ## template.spec will be modified if you change the container mode
+  ## with containerMode.type=dind, we will populate the template.spec with following pod spec
+  ## template:
+  ##   spec:
+  ##     initContainers:
+  ##     - name: init-dind-externals
+  ##       image: ghcr.io/actions/actions-runner:latest
+  ##       command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"]
+  ##       volumeMounts:
+  ##         - name: dind-externals
+  ##           mountPath: /home/runner/tmpDir
+  ##     containers:
+  ##     - name: runner
+  ##       image: ghcr.io/actions/actions-runner:latest
+  ##       command: ["/home/runner/run.sh"]
+  ##       env:
+  ##         - name: DOCKER_HOST
+  ##           value: unix:///run/docker/docker.sock
+  ##       volumeMounts:
+  ##         - name: work
+  ##           mountPath: /home/runner/_work
+  ##         - name: dind-sock
+  ##           mountPath: /run/docker
+  ##           readOnly: true
+  ##     - name: dind
+  ##       image: docker:dind
+  ##       args:
+  ##         - dockerd
+  ##         - --host=unix:///run/docker/docker.sock
+  ##         - --group=$(DOCKER_GROUP_GID)
+  ##       env:
+  ##         - name: DOCKER_GROUP_GID
+  ##           value: "123"
+  ##       securityContext:
+  ##         privileged: true
+  ##       volumeMounts:
+  ##         - name: work
+  ##           mountPath: /home/runner/_work
+  ##         - name: dind-sock
+  ##           mountPath: /run/docker
+  ##         - name: dind-externals
+  ##           mountPath: /home/runner/externals
+  ##     volumes:
+  ##     - name: work
+  ##       emptyDir: {}
+  ##     - name: dind-sock
+  ##       emptyDir: {}
+  ##     - name: dind-externals
+  ##       emptyDir: {}
+  ######################################################################################################
+  ## with containerMode.type=kubernetes, we will populate the template.spec with following pod spec
+template:
+  spec:
+    containers:
+    - name: runner
+      image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest
+      command: ["/bin/bash"]
+      args: ["-c", "/entrypoint.sh /home/runner/run.sh"]
+      env:
+        - name: ACTIONS_RUNNER_CONTAINER_HOOKS
+          value: /home/runner/k8s/index.js
+        - name: ACTIONS_RUNNER_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER
+          value: "false"
+        - name: WORKDIR
+          value: "/home/runner/_work"
+      resources:
+        limits:
+          cpu: 31
+          memory: 80Gi
+        requests:
+          cpu: 31
+          memory: 80Gi
+      volumeMounts:
+        - name: work
+          mountPath: /home/runner/_work
+    volumes:
+      - name: work
+        emptyDir: {}
+    nodeSelector:
+        cloud.google.com/gke-nodepool: linux-agents-2
+
+## Optional controller service account that needs to have required Role and RoleBinding
+## to operate this gha-runner-scale-set installation.
+## The helm chart will try to find the controller deployment and its service account at installation time.
+## In case the helm chart can't find the right service account, you can explicitly pass in the following value
+## to help it finish RoleBinding with the right service account.
+## Note: if your controller is installed to only watch a single namespace, you have to pass these values explicitly.
+# controllerServiceAccount:
+#   namespace: arc-system
+#   name: test-arc-gha-runner-scale-set-controller
--- a/containers/github-linux/Dockerfile
+++ b/containers/github-linux/Dockerfile
@ -6,7 +6,7 @@ RUN echo 'intall packages'; \
    apt-get install -y --no-install-recommends \
        gosu \
        locales openssh-client gnupg ca-certificates  \
-        zip wget curl git \
+        zip unzip wget curl git \
        gdb build-essential  \
        ninja-build \
        libelf-dev libffi-dev gcc-multilib libmpfr-dev libpfm4-dev \
@ -65,21 +65,41 @@ RUN curl -o sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz -L https://github.co
    && ls -la /usr/bin/sccache \
    && sccache --version

-WORKDIR /actions-runner
-
-RUN cd /actions-runner \
-    && curl -o actions-runner-linux-x64-2.308.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.308.0/actions-runner-linux-x64-2.308.0.tar.gz \
-    && echo "9f994158d49c5af39f57a65bf1438cbae4968aec1e4fec132dd7992ad57c74fa  actions-runner-linux-x64-2.308.0.tar.gz" | shasum -a 256 -c \
-    && tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz
-
 RUN groupadd -g 121 runner \
    && useradd -mr -d /home/runner -u 1001 -g 121 runner \
-    && mkdir -p /_work \
-    && chown -R runner:runner /_work /actions-runner;
+    && mkdir -p /home/runner/_work

-COPY entrypoint.sh token.sh /
-RUN chmod +x /entrypoint.sh /token.sh
-# try: USER runner instead of gosu
-ENTRYPOINT ["/entrypoint.sh"]
+WORKDIR /home/runner

-CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"]
+# https://github.com/actions/runner/releases
+ARG RUNNER_VERSION="2.311.0"
+ARG RUNNER_ARCH="x64"
+
+RUN curl -f -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz \
+    && tar xzf ./runner.tar.gz \
+    && rm runner.tar.gz
+
+# https://github.com/actions/runner-container-hooks/releases
+ARG RUNNER_CONTAINER_HOOKS_VERSION="0.4.0"
+RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-container-hooks/releases/download/v${RUNNER_CONTAINER_HOOKS_VERSION}/actions-runner-hooks-k8s-${RUNNER_CONTAINER_HOOKS_VERSION}.zip \
+    && unzip ./runner-container-hooks.zip -d ./k8s \
+    && rm runner-container-hooks.zip
+
+RUN chown -R runner:runner /home/runner
+
+# Copy entrypoint but script. There is no point of adding
+# `ENTRYPOINT ["/entrypoint.sh"]` as k8s runs it with explicit command and
+# entrypoint is ignored.
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+# USER runner
+
+# We don't need entry point and token logic as it's all orchestrated by action runner controller.
+# BUT we need to start server etc
+
+
+# RUN chmod +x /entrypoint.sh /token.sh
+# # try: USER runner instead of gosu
+# ENTRYPOINT ["/entrypoint.sh"]
+# CMD ["sleep", "infinity"]
+# CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"]
--- a/containers/github-linux/entrypoint.sh
+++ b/containers/github-linux/entrypoint.sh
@ -1,6 +1,6 @@
 #!/usr/bin/env bash

-# Copyright 2021 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
 # you may not use this file except in compliance with the License.
@ -15,10 +15,10 @@
 # limitations under the License.
 set -ueo pipefail

-export PATH=${PATH}:/actions-runner
+export PATH=${PATH}:/home/runner

 USER=runner
-WORKDIR=${WORKDIR:-/_work}
+WORKDIR=${WORKDIR:-/home/runner/_work}

 export SCCACHE_DIR="${WORKDIR}/sccache"
 mkdir -p "${SCCACHE_DIR}"
@ -27,56 +27,7 @@ chmod oug+rw "${SCCACHE_DIR}"
 gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server'
 sccache --show-stats

-# Configure github runner. TODO: move to a separate file.
-# Based on https://github.com/myoung34/docker-github-actions-runner/blob/master/entrypoint.sh
-# licensed under MIT https://github.com/myoung34/docker-github-actions-runner/blob/master/LICENSE
-export -n ACCESS_TOKEN
-RUNNER_SCOPE=${RUNNER_SCOPE:-repo}
-RUNNER_SCOPE="${RUNNER_SCOPE,,}" # to lowercase
-_GITHUB_HOST=${GITHUB_HOST:="github.com"}
-case ${RUNNER_SCOPE} in
-  org*)
-    [[ -z ${ORG_NAME} ]] && ( echo "ORG_NAME required for org runners"; exit 1 )
-    _SHORT_URL="https://${_GITHUB_HOST}/${ORG_NAME}"
-    RUNNER_SCOPE="org"
-    ;;
-
-  ent*)
-    [[ -z ${ENTERPRISE_NAME} ]] && ( echo "ENTERPRISE_NAME required for enterprise runners"; exit 1 )
-    _SHORT_URL="https://${_GITHUB_HOST}/enterprises/${ENTERPRISE_NAME}"
-    RUNNER_SCOPE="enterprise"
-    ;;
-
-  *)
-    [[ -z ${REPO_URL} ]] && ( echo "REPO_URL required for repo runners"; exit 1 )
-    _SHORT_URL=${REPO_URL}
-    RUNNER_SCOPE="repo"
-    ;;
-esac
-_RUNNER_NAME=${RUNNER_NAME:-${RUNNER_NAME_PREFIX:-github-runner}-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo '')}
-_LABELS=${LABELS:-default}
-echo "Configuring"
-echo "runner URL" "${_SHORT_URL}"
-echo "workdir ${WORKDIR}"
-echo "access token" "${ACCESS_TOKEN}"
-echo "labels ${_LABELS}"
-echo "runner name" "${_RUNNER_NAME}"
-
-echo "Obtaining the token of the runner"
-_TOKEN=$(ACCESS_TOKEN="${ACCESS_TOKEN}" bash /token.sh)
-RUNNER_TOKEN=$(echo "${_TOKEN}" | jq -r .token)
-echo "RUNNER_TOKEN ${RUNNER_TOKEN}"
-
-gosu runner ./config.sh \
-    --url "${_SHORT_URL}" \
-    --token "${RUNNER_TOKEN}" \
-    --name "${_RUNNER_NAME}" \
-    --work "${WORKDIR}" \
-    --labels "${_LABELS}" \
-    --unattended \
-    --replace
-
-[[ ! -d "${WORKDIR}" ]] && mkdir "${WORKDIR}"
+[[ ! -d "${WORKDIR}" ]] && mkdir -p "${WORKDIR}"

 # exec /usr/bin/tini -g -- $@
 gosu runner "$@"