deployment configs

containers/stats/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:unstable
+FROM ubuntu:latest
 
 RUN apt-get update ;\
     apt-get upgrade -y;\
@@ -7,21 +7,23 @@ RUN apt-get update ;\
         build-essential \
         zip wget git less vim \
         python3 python3-psutil python3-pip python3-setuptools pipenv \
-        python3 python3-psutil python3-pip python3-setuptools pipenv \
+        # PostgreSQL
+        libpq-dev \
+        # debugging
+        less vim wget curl \
         rsync jq tini gosu;
 
-
-RUN echo 'configure locale'; \
-    sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
-    locale-gen ;
+RUN echo 'configure locale' && \
+    sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen && \
+    locale-gen
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
 COPY *.sh /usr/local/bin/
 # requirements.txt generated by running `pipenv lock -r > ../../containers/stats/requirements.txt` in `scripts/metrics`.
-COPY requirements.txt /tmp/
-RUN pip install -q -r /tmp/requirements.txt
+# COPY requirements.txt /tmp/
+# RUN pip install -q -r /tmp/requirements.txt
 RUN chmod og+rx /usr/local/bin/*.sh
 RUN wget -q https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/local/bin/cloud_sql_proxy;\
     chmod +x /usr/local/bin/cloud_sql_proxy

kubernetes/buildkite/linux-112.yaml

@@ -1,77 +0,0 @@
-# Copyright 2023 Google LLC
-#
-# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://llvm.org/LICENSE.txt
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: linux-test-112
-  namespace: buildkite
-spec:
-  replicas: 1
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 1
-      maxSurge: 0
-    type: RollingUpdate
-  selector:
-    matchLabels:
-      app: linux-test-112
-  template:
-    metadata:
-      labels:
-        app: linux-test-112
-    spec:
-      containers:
-        - name: buildkite-linux
-          image: gcr.io/llvm-premerge-checks/buildkite-linux:latest
-          resources:
-            limits:
-              cpu: 110
-              memory: 420Gi
-            requests:
-              cpu: 110
-              memory: 420Gi
-          volumeMounts:
-            - name: github-ssh
-              mountPath: /mnt/ssh
-            - name: workdir
-              mountPath: /var/lib/buildkite-agent
-          env:
-            - name: BUILDKITE_AGENT_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: buildkite-agent-token
-                  key: token
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                 fieldPath: metadata.name
-            - name: BUILDKITE_AGENT_TAGS
-              value: "queue=linux-test-112,name=$(POD_NAME),cluster=google"
-            - name: BUILDKITE_BUILD_PATH
-              value: "/var/lib/buildkite-agent/builds"
-            - name: BUILDKITE_API_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: buildkite-api-token-readonly
-                  key: token
-      volumes:
-        - name: github-ssh
-          secret:
-            secretName: github-ssh
-        - name: workdir
-          emptyDir: {}
-      nodeSelector:
-        cloud.google.com/gke-nodepool: linux-big
-      terminationGracePeriodSeconds: 30

kubernetes/buildkite/linux-test.yaml

@@ -26,11 +26,11 @@ spec:
     type: RollingUpdate
   selector:
     matchLabels:
-      app: linux-test-32
+      app: linux-test
   template:
     metadata:
       labels:
-        app: linux-test-32
+        app: linux-test
     spec:
       containers:
         - name: runner
@@ -58,7 +58,7 @@ spec:
                 fieldRef:
                  fieldPath: metadata.name
             - name: BUILDKITE_AGENT_TAGS
-              value: "queue=linux-test-32,name=$(POD_NAME),cluster=google"
+              value: "queue=linux-test,name=$(POD_NAME),cluster=google,node-pool=linux-agents-2"
             - name: BUILDKITE_BUILD_PATH
               value: "/var/lib/buildkite-agent/builds"
             - name: CONDUIT_TOKEN
@@ -78,5 +78,5 @@ spec:
         - name: workdir
           emptyDir: {}
       nodeSelector:
-        cloud.google.com/gke-nodepool: linux-32
+        cloud.google.com/gke-nodepool: linux-agents-2
       terminationGracePeriodSeconds: 30

kubernetes/cron/buildkite.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: buildkite-stats
@@ -60,4 +60,4 @@ spec:
               value: "main"
           restartPolicy: Never
           nodeSelector:
-            cloud.google.com/gke-nodepool: service
+            cloud.google.com/gke-nodepool: default-pool

scripts/metrics/download_buildkite_builds_pg.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+echo "downloading buildkite builds..."
+env