From 8d30467ddee45b5742158aafc5b569f71419fcf0 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Fri, 10 Nov 2023 15:42:18 +0100 Subject: [PATCH] action runners controller setup --- actions-runner-controller/README.md | 29 +++ actions-runner-controller/values-llvm.yaml | 46 +++++ actions-runner-controller/values-test.yaml | 194 +++++++++++++++++++++ containers/github-linux/Dockerfile | 50 ++++-- containers/github-linux/entrypoint.sh | 57 +----- 5 files changed, 308 insertions(+), 68 deletions(-) create mode 100644 actions-runner-controller/README.md create mode 100644 actions-runner-controller/values-llvm.yaml create mode 100644 actions-runner-controller/values-test.yaml diff --git a/actions-runner-controller/README.md b/actions-runner-controller/README.md new file mode 100644 index 0000000..1ab7e77 --- /dev/null +++ b/actions-runner-controller/README.md @@ -0,0 +1,29 @@ +# Installation + +Install helm (along with other tools in `local_setup.sh`). + +Once per cluster: + +`helm install arc --namespace "arc-system" --create-namespace oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller` + +## Add new set of runners + +Create runner set first time: + +- copy 'values.yml` and updata parameters: APP ID, target node set, repo etc +- run + +``` +helm install --namespace --create-namespace \ + --values=$(readlink -f ) + --set-file=githubConfigSecret.github_app_private_key=$(readlink -f ) \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set +``` + +After that update values.yml to use `githubConfigSecret: arc-runner-set-gha-rs-github-secret`. + +## Update + +Example command for linux set: + +`helm upgrade arc-google-linux --namespace arc-linux-prod -f $(readlink -f ~/src/merge-checks/actions-runner-controller/values-llvm.yaml) oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set` diff --git a/actions-runner-controller/values-llvm.yaml b/actions-runner-controller/values-llvm.yaml new file mode 100644 index 0000000..86054ed --- /dev/null +++ b/actions-runner-controller/values-llvm.yaml @@ -0,0 +1,46 @@ +# See full list settings doc in https://github.com/actions/actions-runner-controller/tree/master/charts/actions-runner-controller. + +githubConfigUrl: "https://github.com/llvm/llvm-project" + +# Created by first installation. +githubConfigSecret: arc-runner-set-gha-rs-github-secret + +minRunners: 1 +maxRunners: 3 + +runnerGroup: "generic-google-cloud" + +## template for each runner Pod +template: + spec: + containers: + - name: runner + image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest + command: ["/bin/bash"] + args: ["-c", "/entrypoint.sh /home/runner/run.sh"] + env: + - name: ACTIONS_RUNNER_CONTAINER_HOOKS + value: /home/runner/k8s/index.js + - name: ACTIONS_RUNNER_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER + value: "false" + - name: WORKDIR + value: "/home/runner/_work" + resources: + limits: + cpu: 31 + memory: 80Gi + requests: + cpu: 31 + memory: 80Gi + volumeMounts: + - name: work + mountPath: /home/runner/_work + volumes: + - name: work + emptyDir: {} + nodeSelector: + cloud.google.com/gke-nodepool: linux-agents-2 diff --git a/actions-runner-controller/values-test.yaml b/actions-runner-controller/values-test.yaml new file mode 100644 index 0000000..bf40bb3 --- /dev/null +++ b/actions-runner-controller/values-test.yaml @@ -0,0 +1,194 @@ +# See options doc in https://github.com/actions/actions-runner-controller/tree/master/charts/actions-runner-controller + +## githubConfigUrl is the GitHub url for where you want to configure runners +## ex: https://github.com/myorg/myrepo or https://github.com/myorg +githubConfigUrl: "https://github.com/metafloworg/llvm-project" + +## githubConfigSecret is the k8s secrets to use when auth with GitHub API. +#githubConfigSecret: + ### GitHub Apps Configuration + ## NOTE: IDs MUST be strings, use quotes + #github_app_id: "427039" + #github_app_installation_id: "43840704" + ## Pass --set-file=githubConfigSecret.github_app_private_key= +# First installation creates this secret. +githubConfigSecret: arc-runner-set-gha-rs-github-secret + +## proxy can be used to define proxy settings that will be used by the +## controller, the listener and the runner of this scale set. +# +# proxy: +# http: +# url: http://proxy.com:1234 +# credentialSecretRef: proxy-auth # a secret with `username` and `password` keys +# https: +# url: http://proxy.com:1234 +# credentialSecretRef: proxy-auth # a secret with `username` and `password` keys +# noProxy: +# - example.com +# - example.org + +## maxRunners is the max number of runners the autoscaling runner set will scale up to. +maxRunners: 3 + +## minRunners is the min number of runners the autoscaling runner set will scale down to. +minRunners: 1 + +# runnerGroup: "default" + +## name of the runner scale set to create. Defaults to the helm release name +# runnerScaleSetName: "" + +## A self-signed CA certificate for communication with the GitHub server can be +## provided using a config map key selector. If `runnerMountPath` is set, for +## each runner pod ARC will: +## - create a `github-server-tls-cert` volume containing the certificate +## specified in `certificateFrom` +## - mount that volume on path `runnerMountPath`/{certificate name} +## - set NODE_EXTRA_CA_CERTS environment variable to that same path +## - set RUNNER_UPDATE_CA_CERTS environment variable to "1" (as of version +## 2.303.0 this will instruct the runner to reload certificates on the host) +## +## If any of the above had already been set by the user in the runner pod +## template, ARC will observe those and not overwrite them. +## Example configuration: +# +# githubServerTLS: +# certificateFrom: +# configMapKeyRef: +# name: config-map-name +# key: ca.crt +# runnerMountPath: /usr/local/share/ca-certificates/ + +## Container mode is an object that provides out-of-box configuration +## for dind and kubernetes mode. Template will be modified as documented under the +## template object. +## +## If any customization is required for dind or kubernetes mode, containerMode should remain +## empty, and configuration should be applied to the template. +# containerMode: +# type: "dind" ## type can be set to dind or kubernetes +# ## the following is required when containerMode.type=kubernetes +# kubernetesModeWorkVolumeClaim: +# accessModes: ["ReadWriteOnce"] +# # For local testing, use https://github.com/openebs/dynamic-localpv-provisioner/blob/develop/docs/quickstart.md to provide dynamic provision volume with storageClassName: openebs-hostpath +# storageClassName: "dynamic-blob-storage" +# resources: +# requests: +# storage: 1Gi +# kubernetesModeServiceAccount: +# annotations: + +## template is the PodSpec for each listener Pod +## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec +# listenerTemplate: +# spec: +# containers: +# # Use this section to append additional configuration to the listener container. +# # If you change the name of the container, the configuration will not be applied to the listener, +# # and it will be treated as a side-car container. +# - name: listener +# securityContext: +# runAsUser: 1000 +# # Use this section to add the configuration of a side-car container. +# # Comment it out or remove it if you don't need it. +# # Spec for this container will be applied as is without any modifications. +# - name: side-car +# image: example-sidecar + +## template is the PodSpec for each runner Pod +## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec + ## template.spec will be modified if you change the container mode + ## with containerMode.type=dind, we will populate the template.spec with following pod spec + ## template: + ## spec: + ## initContainers: + ## - name: init-dind-externals + ## image: ghcr.io/actions/actions-runner:latest + ## command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"] + ## volumeMounts: + ## - name: dind-externals + ## mountPath: /home/runner/tmpDir + ## containers: + ## - name: runner + ## image: ghcr.io/actions/actions-runner:latest + ## command: ["/home/runner/run.sh"] + ## env: + ## - name: DOCKER_HOST + ## value: unix:///run/docker/docker.sock + ## volumeMounts: + ## - name: work + ## mountPath: /home/runner/_work + ## - name: dind-sock + ## mountPath: /run/docker + ## readOnly: true + ## - name: dind + ## image: docker:dind + ## args: + ## - dockerd + ## - --host=unix:///run/docker/docker.sock + ## - --group=$(DOCKER_GROUP_GID) + ## env: + ## - name: DOCKER_GROUP_GID + ## value: "123" + ## securityContext: + ## privileged: true + ## volumeMounts: + ## - name: work + ## mountPath: /home/runner/_work + ## - name: dind-sock + ## mountPath: /run/docker + ## - name: dind-externals + ## mountPath: /home/runner/externals + ## volumes: + ## - name: work + ## emptyDir: {} + ## - name: dind-sock + ## emptyDir: {} + ## - name: dind-externals + ## emptyDir: {} + ###################################################################################################### + ## with containerMode.type=kubernetes, we will populate the template.spec with following pod spec +template: + spec: + containers: + - name: runner + image: us-central1-docker.pkg.dev/llvm-premerge-checks/docker/github-linux:latest + command: ["/bin/bash"] + args: ["-c", "/entrypoint.sh /home/runner/run.sh"] + env: + - name: ACTIONS_RUNNER_CONTAINER_HOOKS + value: /home/runner/k8s/index.js + - name: ACTIONS_RUNNER_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER + value: "false" + - name: WORKDIR + value: "/home/runner/_work" + resources: + limits: + cpu: 31 + memory: 80Gi + requests: + cpu: 31 + memory: 80Gi + volumeMounts: + - name: work + mountPath: /home/runner/_work + volumes: + - name: work + emptyDir: {} + nodeSelector: + cloud.google.com/gke-nodepool: linux-agents-2 + +## Optional controller service account that needs to have required Role and RoleBinding +## to operate this gha-runner-scale-set installation. +## The helm chart will try to find the controller deployment and its service account at installation time. +## In case the helm chart can't find the right service account, you can explicitly pass in the following value +## to help it finish RoleBinding with the right service account. +## Note: if your controller is installed to only watch a single namespace, you have to pass these values explicitly. +# controllerServiceAccount: +# namespace: arc-system +# name: test-arc-gha-runner-scale-set-controller diff --git a/containers/github-linux/Dockerfile b/containers/github-linux/Dockerfile index 1cee7d3..453eeb1 100644 --- a/containers/github-linux/Dockerfile +++ b/containers/github-linux/Dockerfile @@ -6,7 +6,7 @@ RUN echo 'intall packages'; \ apt-get install -y --no-install-recommends \ gosu \ locales openssh-client gnupg ca-certificates \ - zip wget curl git \ + zip unzip wget curl git \ gdb build-essential \ ninja-build \ libelf-dev libffi-dev gcc-multilib libmpfr-dev libpfm4-dev \ @@ -65,21 +65,41 @@ RUN curl -o sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz -L https://github.co && ls -la /usr/bin/sccache \ && sccache --version -WORKDIR /actions-runner - -RUN cd /actions-runner \ - && curl -o actions-runner-linux-x64-2.308.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.308.0/actions-runner-linux-x64-2.308.0.tar.gz \ - && echo "9f994158d49c5af39f57a65bf1438cbae4968aec1e4fec132dd7992ad57c74fa actions-runner-linux-x64-2.308.0.tar.gz" | shasum -a 256 -c \ - && tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz - RUN groupadd -g 121 runner \ && useradd -mr -d /home/runner -u 1001 -g 121 runner \ - && mkdir -p /_work \ - && chown -R runner:runner /_work /actions-runner; + && mkdir -p /home/runner/_work -COPY entrypoint.sh token.sh / -RUN chmod +x /entrypoint.sh /token.sh -# try: USER runner instead of gosu -ENTRYPOINT ["/entrypoint.sh"] +WORKDIR /home/runner -CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] +# https://github.com/actions/runner/releases +ARG RUNNER_VERSION="2.311.0" +ARG RUNNER_ARCH="x64" + +RUN curl -f -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz \ + && tar xzf ./runner.tar.gz \ + && rm runner.tar.gz + +# https://github.com/actions/runner-container-hooks/releases +ARG RUNNER_CONTAINER_HOOKS_VERSION="0.4.0" +RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-container-hooks/releases/download/v${RUNNER_CONTAINER_HOOKS_VERSION}/actions-runner-hooks-k8s-${RUNNER_CONTAINER_HOOKS_VERSION}.zip \ + && unzip ./runner-container-hooks.zip -d ./k8s \ + && rm runner-container-hooks.zip + +RUN chown -R runner:runner /home/runner + +# Copy entrypoint but script. There is no point of adding +# `ENTRYPOINT ["/entrypoint.sh"]` as k8s runs it with explicit command and +# entrypoint is ignored. +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +# USER runner + +# We don't need entry point and token logic as it's all orchestrated by action runner controller. +# BUT we need to start server etc + + +# RUN chmod +x /entrypoint.sh /token.sh +# # try: USER runner instead of gosu +# ENTRYPOINT ["/entrypoint.sh"] +# CMD ["sleep", "infinity"] +# CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] diff --git a/containers/github-linux/entrypoint.sh b/containers/github-linux/entrypoint.sh index 97a3c4c..85f35e2 100755 --- a/containers/github-linux/entrypoint.sh +++ b/containers/github-linux/entrypoint.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright 2021 Google LLC +# Copyright 2023 Google LLC # # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); # you may not use this file except in compliance with the License. @@ -15,10 +15,10 @@ # limitations under the License. set -ueo pipefail -export PATH=${PATH}:/actions-runner +export PATH=${PATH}:/home/runner USER=runner -WORKDIR=${WORKDIR:-/_work} +WORKDIR=${WORKDIR:-/home/runner/_work} export SCCACHE_DIR="${WORKDIR}/sccache" mkdir -p "${SCCACHE_DIR}" @@ -27,56 +27,7 @@ chmod oug+rw "${SCCACHE_DIR}" gosu runner bash -c 'SCCACHE_DIR="${SCCACHE_DIR}" SCCACHE_IDLE_TIMEOUT=0 SCCACHE_CACHE_SIZE=20G sccache --start-server' sccache --show-stats -# Configure github runner. TODO: move to a separate file. -# Based on https://github.com/myoung34/docker-github-actions-runner/blob/master/entrypoint.sh -# licensed under MIT https://github.com/myoung34/docker-github-actions-runner/blob/master/LICENSE -export -n ACCESS_TOKEN -RUNNER_SCOPE=${RUNNER_SCOPE:-repo} -RUNNER_SCOPE="${RUNNER_SCOPE,,}" # to lowercase -_GITHUB_HOST=${GITHUB_HOST:="github.com"} -case ${RUNNER_SCOPE} in - org*) - [[ -z ${ORG_NAME} ]] && ( echo "ORG_NAME required for org runners"; exit 1 ) - _SHORT_URL="https://${_GITHUB_HOST}/${ORG_NAME}" - RUNNER_SCOPE="org" - ;; - - ent*) - [[ -z ${ENTERPRISE_NAME} ]] && ( echo "ENTERPRISE_NAME required for enterprise runners"; exit 1 ) - _SHORT_URL="https://${_GITHUB_HOST}/enterprises/${ENTERPRISE_NAME}" - RUNNER_SCOPE="enterprise" - ;; - - *) - [[ -z ${REPO_URL} ]] && ( echo "REPO_URL required for repo runners"; exit 1 ) - _SHORT_URL=${REPO_URL} - RUNNER_SCOPE="repo" - ;; -esac -_RUNNER_NAME=${RUNNER_NAME:-${RUNNER_NAME_PREFIX:-github-runner}-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo '')} -_LABELS=${LABELS:-default} -echo "Configuring" -echo "runner URL" "${_SHORT_URL}" -echo "workdir ${WORKDIR}" -echo "access token" "${ACCESS_TOKEN}" -echo "labels ${_LABELS}" -echo "runner name" "${_RUNNER_NAME}" - -echo "Obtaining the token of the runner" -_TOKEN=$(ACCESS_TOKEN="${ACCESS_TOKEN}" bash /token.sh) -RUNNER_TOKEN=$(echo "${_TOKEN}" | jq -r .token) -echo "RUNNER_TOKEN ${RUNNER_TOKEN}" - -gosu runner ./config.sh \ - --url "${_SHORT_URL}" \ - --token "${RUNNER_TOKEN}" \ - --name "${_RUNNER_NAME}" \ - --work "${WORKDIR}" \ - --labels "${_LABELS}" \ - --unattended \ - --replace - -[[ ! -d "${WORKDIR}" ]] && mkdir "${WORKDIR}" +[[ ! -d "${WORKDIR}" ]] && mkdir -p "${WORKDIR}" # exec /usr/bin/tini -g -- $@ gosu runner "$@"