From 08bb6492fa4cc6d3aa58d360407d8e8bfc5087d7 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Mon, 26 Apr 2021 20:10:15 +0200 Subject: [PATCH] service queue --- kubernetes/buildkite/service-agents.yaml | 75 ++++++++++++++++++++++++ scripts/pipeline_create_branch.py | 2 +- scripts/pipeline_premerge.py | 4 +- scripts/steps.py | 28 ++++----- 4 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 kubernetes/buildkite/service-agents.yaml diff --git a/kubernetes/buildkite/service-agents.yaml b/kubernetes/buildkite/service-agents.yaml new file mode 100644 index 0000000..3ec1f26 --- /dev/null +++ b/kubernetes/buildkite/service-agents.yaml @@ -0,0 +1,75 @@ +# Copyright 2019 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: service-agents + namespace: buildkite +spec: + replicas: 4 + strategy: + rollingUpdate: + maxSurge: 50% + maxUnavailable: 50% + type: RollingUpdate + selector: + matchLabels: + app: agent-premerge-debian + template: + metadata: + labels: + app: agent-premerge-debian + spec: + containers: + - name: buildkite-premerge-debian + image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian + resources: + limits: + cpu: 2 + memory: 5Gi + requests: + cpu: 1.5 + memory: 5Gi + volumeMounts: + - name: github-ssh + mountPath: /mnt/ssh + env: + - name: BUILDKITE_AGENT_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-agent-token + key: token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: BUILDKITE_AGENT_TAGS + value: "queue=service,name=$(POD_NAME)" + - name: CONDUIT_TOKEN + valueFrom: + secretKeyRef: + name: conduit-api-token + key: token + - name: BUILDKITE_API_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-api-token-readonly + key: token + volumes: + - name: github-ssh + secret: + secretName: github-ssh + nodeSelector: + cloud.google.com/gke-nodepool: default-pool \ No newline at end of file diff --git a/scripts/pipeline_create_branch.py b/scripts/pipeline_create_branch.py index c25b4e8..85afcfe 100755 --- a/scripts/pipeline_create_branch.py +++ b/scripts/pipeline_create_branch.py @@ -33,7 +33,7 @@ if __name__ == '__main__': 'pip install -q -r scripts/requirements.txt', 'scripts/apply_patch.sh' ], - 'agents': {'queue': 'linux'}, + 'agents': {'queue': 'service'}, 'timeout_in_minutes': 20, 'env': { 'LOG_LEVEL': log_level, diff --git a/scripts/pipeline_premerge.py b/scripts/pipeline_premerge.py index f74e3d6..e1c2443 100755 --- a/scripts/pipeline_premerge.py +++ b/scripts/pipeline_premerge.py @@ -84,10 +84,10 @@ if __name__ == '__main__': 'label': ':phabricator: update build status on Phabricator', 'commands': [ *checkout_scripts('linux', scripts_refspec), - '${SRC}/scripts/summary.py', + '$${SRC}/scripts/summary.py', ], 'artifact_paths': ['artifacts/**/*'], - 'agents': {'queue': 'linux'}, + 'agents': {'queue': 'service'}, 'timeout_in_minutes': 10, } steps.append(report_step) diff --git a/scripts/steps.py b/scripts/steps.py index f6834e8..c71a8cd 100644 --- a/scripts/steps.py +++ b/scripts/steps.py @@ -46,19 +46,19 @@ def generic_linux(projects: str, check_diff: bool) -> List: if check_diff: commands.extend([ - '${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy ' + '$${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy ' f'--projects="{projects}" --log-level={log_level}', ]) else: commands.extend([ - f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}' + f'$${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}' ]) commands.extend([ - 'EXIT_STATUS=\\$?', + 'EXIT_STATUS=$$?', 'echo "--- ccache stats"', 'ccache --print-stats', 'ccache --show-stats', - 'exit \\$EXIT_STATUS', + 'exit $$EXIT_STATUS', ]) linux_buld_step = { @@ -82,8 +82,8 @@ def generic_windows(projects: str) -> List: scripts_refspec = os.getenv("ph_scripts_refspec", "main") no_cache = os.getenv('ph_no_cache') is not None log_level = os.getenv('ph_log_level', 'WARNING') - clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \ - 'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \ + clear_sccache = 'powershell -command "sccache --stop-server; echo $$env:SCCACHE_DIR; ' \ + 'Remove-Item -Recurse -Force -ErrorAction Ignore $$env:SCCACHE_DIR; ' \ 'sccache --start-server"' win_agents = {'queue': 'windows'} t = os.getenv('ph_windows_agents') @@ -99,9 +99,9 @@ def generic_windows(projects: str) -> List: 'powershell -command "' f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level}; ' - '\\$exit=\\$?;' + '$$exit=$$?;' 'sccache --show-stats;' - 'if (\\$exit) {' + 'if ($$exit) {' ' echo success;' ' exit 0; } ' 'else {' @@ -166,14 +166,14 @@ def checkout_scripts(target_os: str, scripts_refspec: str) -> []: 'cd %BUILDKITE_BUILD_CHECKOUT_PATH%', ] return [ - 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', - 'rm -rf ${SRC}', - 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"', - 'cd ${SRC}', + 'export SRC=$${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', + 'rm -rf $${SRC}', + 'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "$${SRC}"', + 'cd $${SRC}', f'git fetch origin "{scripts_refspec}":x', 'git checkout x', 'echo "llvm-premerge-checks commit"', 'git rev-parse HEAD', - 'pip install -q -r ${SRC}/scripts/requirements.txt', - 'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"', + 'pip install -q -r $${SRC}/scripts/requirements.txt', + 'cd "$$BUILDKITE_BUILD_CHECKOUT_PATH"', ]