service queue

kubernetes/buildkite/service-agents.yaml

@@ -0,0 +1,75 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: service-agents
+  namespace: buildkite
+spec:
+  replicas: 4
+  strategy:
+    rollingUpdate:
+      maxSurge: 50%
+      maxUnavailable: 50%
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: agent-premerge-debian
+  template:
+    metadata:
+      labels:
+        app: agent-premerge-debian
+    spec:
+      containers:
+        - name: buildkite-premerge-debian
+          image: gcr.io/llvm-premerge-checks/buildkite-premerge-debian
+          resources:
+            limits:
+              cpu: 2
+              memory: 5Gi
+            requests:
+              cpu: 1.5
+              memory: 5Gi
+          volumeMounts:
+            - name: github-ssh
+              mountPath: /mnt/ssh
+          env:
+            - name: BUILDKITE_AGENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: buildkite-agent-token
+                  key: token
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                 fieldPath: metadata.name
+            - name: BUILDKITE_AGENT_TAGS
+              value: "queue=service,name=$(POD_NAME)"
+            - name: CONDUIT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: conduit-api-token
+                  key: token
+            - name: BUILDKITE_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: buildkite-api-token-readonly
+                  key: token
+      volumes:
+        - name: github-ssh
+          secret:
+            secretName: github-ssh
+      nodeSelector:
+        cloud.google.com/gke-nodepool: default-pool

scripts/pipeline_create_branch.py

@@ -33,7 +33,7 @@ if __name__ == '__main__':
             'pip install -q -r scripts/requirements.txt',
             'scripts/apply_patch.sh'
         ],
-        'agents': {'queue': 'linux'},
+        'agents': {'queue': 'service'},
         'timeout_in_minutes': 20,
         'env': {
             'LOG_LEVEL': log_level,

scripts/pipeline_premerge.py

@@ -84,10 +84,10 @@ if __name__ == '__main__':
             'label': ':phabricator: update build status on Phabricator',
             'commands': [
                 *checkout_scripts('linux', scripts_refspec),
-                '${SRC}/scripts/summary.py',
+                '$${SRC}/scripts/summary.py',
             ],
             'artifact_paths': ['artifacts/**/*'],
-            'agents': {'queue': 'linux'},
+            'agents': {'queue': 'service'},
             'timeout_in_minutes': 10,
         }
         steps.append(report_step)

scripts/steps.py

@@ -46,19 +46,19 @@ def generic_linux(projects: str, check_diff: bool) -> List:
 
     if check_diff:
         commands.extend([
-            '${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy '
+            '$${SRC}/scripts/premerge_checks.py --check-clang-format --check-clang-tidy '
             f'--projects="{projects}" --log-level={log_level}',
         ])
     else:
         commands.extend([
-            f'${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}'
+            f'$${{SRC}}/scripts/premerge_checks.py --projects="{projects}" --log-level={log_level}'
         ])
     commands.extend([
-        'EXIT_STATUS=\\$?',
+        'EXIT_STATUS=$$?',
         'echo "--- ccache stats"',
         'ccache --print-stats',
         'ccache --show-stats',
-        'exit \\$EXIT_STATUS',
+        'exit $$EXIT_STATUS',
     ])
 
     linux_buld_step = {
@@ -82,8 +82,8 @@ def generic_windows(projects: str) -> List:
     scripts_refspec = os.getenv("ph_scripts_refspec", "main")
     no_cache = os.getenv('ph_no_cache') is not None
     log_level = os.getenv('ph_log_level', 'WARNING')
-    clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \
+    clear_sccache = 'powershell -command "sccache --stop-server; echo $$env:SCCACHE_DIR; ' \
-                    'Remove-Item -Recurse -Force -ErrorAction Ignore \\$env:SCCACHE_DIR; ' \
+                    'Remove-Item -Recurse -Force -ErrorAction Ignore $$env:SCCACHE_DIR; ' \
                     'sccache --start-server"'
     win_agents = {'queue': 'windows'}
     t = os.getenv('ph_windows_agents')
@@ -99,9 +99,9 @@ def generic_windows(projects: str) -> List:
 
             'powershell -command "'
             f'%SRC%/scripts/premerge_checks.py --projects=\'{projects}\' --log-level={log_level}; '
-            '\\$exit=\\$?;'
+            '$$exit=$$?;'
             'sccache --show-stats;'
-            'if (\\$exit) {'
+            'if ($$exit) {'
             '  echo success;'
             '  exit 0; } '
             'else {'
@@ -166,14 +166,14 @@ def checkout_scripts(target_os: str, scripts_refspec: str) -> []:
             'cd %BUILDKITE_BUILD_CHECKOUT_PATH%',
         ]
     return [
-        'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
+        'export SRC=$${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
-        'rm -rf ${SRC}',
+        'rm -rf $${SRC}',
-        'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}"',
+        'git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "$${SRC}"',
-        'cd ${SRC}',
+        'cd $${SRC}',
         f'git fetch origin "{scripts_refspec}":x',
         'git checkout x',
         'echo "llvm-premerge-checks commit"',
         'git rev-parse HEAD',
-        'pip install -q -r ${SRC}/scripts/requirements.txt',
+        'pip install -q -r $${SRC}/scripts/requirements.txt',
-        'cd "$BUILDKITE_BUILD_CHECKOUT_PATH"',
+        'cd "$$BUILDKITE_BUILD_CHECKOUT_PATH"',
     ]