From d57be337f9f14f10a224f47b6bfa9c68acf65339 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Mon, 4 Sep 2023 18:19:39 +0200 Subject: [PATCH] deployment configs --- containers/stats/Dockerfile | 18 +++-- kubernetes/buildkite/linux-112.yaml | 77 ------------------- .../{linux-32.yaml => linux-test.yaml} | 8 +- kubernetes/cron/buildkite.yaml | 4 +- .../metrics/download_buildkite_builds_pg.sh | 7 ++ 5 files changed, 23 insertions(+), 91 deletions(-) delete mode 100644 kubernetes/buildkite/linux-112.yaml rename kubernetes/buildkite/{linux-32.yaml => linux-test.yaml} (92%) create mode 100644 scripts/metrics/download_buildkite_builds_pg.sh diff --git a/containers/stats/Dockerfile b/containers/stats/Dockerfile index 664e837..8a7fbae 100644 --- a/containers/stats/Dockerfile +++ b/containers/stats/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:unstable +FROM ubuntu:latest RUN apt-get update ;\ apt-get upgrade -y;\ @@ -7,21 +7,23 @@ RUN apt-get update ;\ build-essential \ zip wget git less vim \ python3 python3-psutil python3-pip python3-setuptools pipenv \ - python3 python3-psutil python3-pip python3-setuptools pipenv \ + # PostgreSQL + libpq-dev \ + # debugging + less vim wget curl \ rsync jq tini gosu; - -RUN echo 'configure locale'; \ - sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ - locale-gen ; +RUN echo 'configure locale' && \ + sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen && \ + locale-gen ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 COPY *.sh /usr/local/bin/ # requirements.txt generated by running `pipenv lock -r > ../../containers/stats/requirements.txt` in `scripts/metrics`. -COPY requirements.txt /tmp/ -RUN pip install -q -r /tmp/requirements.txt +# COPY requirements.txt /tmp/ +# RUN pip install -q -r /tmp/requirements.txt RUN chmod og+rx /usr/local/bin/*.sh RUN wget -q https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/local/bin/cloud_sql_proxy;\ chmod +x /usr/local/bin/cloud_sql_proxy diff --git a/kubernetes/buildkite/linux-112.yaml b/kubernetes/buildkite/linux-112.yaml deleted file mode 100644 index 8fcdf71..0000000 --- a/kubernetes/buildkite/linux-112.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: linux-test-112 - namespace: buildkite -spec: - replicas: 1 - strategy: - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - type: RollingUpdate - selector: - matchLabels: - app: linux-test-112 - template: - metadata: - labels: - app: linux-test-112 - spec: - containers: - - name: buildkite-linux - image: gcr.io/llvm-premerge-checks/buildkite-linux:latest - resources: - limits: - cpu: 110 - memory: 420Gi - requests: - cpu: 110 - memory: 420Gi - volumeMounts: - - name: github-ssh - mountPath: /mnt/ssh - - name: workdir - mountPath: /var/lib/buildkite-agent - env: - - name: BUILDKITE_AGENT_TOKEN - valueFrom: - secretKeyRef: - name: buildkite-agent-token - key: token - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: BUILDKITE_AGENT_TAGS - value: "queue=linux-test-112,name=$(POD_NAME),cluster=google" - - name: BUILDKITE_BUILD_PATH - value: "/var/lib/buildkite-agent/builds" - - name: BUILDKITE_API_TOKEN - valueFrom: - secretKeyRef: - name: buildkite-api-token-readonly - key: token - volumes: - - name: github-ssh - secret: - secretName: github-ssh - - name: workdir - emptyDir: {} - nodeSelector: - cloud.google.com/gke-nodepool: linux-big - terminationGracePeriodSeconds: 30 diff --git a/kubernetes/buildkite/linux-32.yaml b/kubernetes/buildkite/linux-test.yaml similarity index 92% rename from kubernetes/buildkite/linux-32.yaml rename to kubernetes/buildkite/linux-test.yaml index 478bd67..b590399 100644 --- a/kubernetes/buildkite/linux-32.yaml +++ b/kubernetes/buildkite/linux-test.yaml @@ -26,11 +26,11 @@ spec: type: RollingUpdate selector: matchLabels: - app: linux-test-32 + app: linux-test template: metadata: labels: - app: linux-test-32 + app: linux-test spec: containers: - name: runner @@ -58,7 +58,7 @@ spec: fieldRef: fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=linux-test-32,name=$(POD_NAME),cluster=google" + value: "queue=linux-test,name=$(POD_NAME),cluster=google,node-pool=linux-agents-2" - name: BUILDKITE_BUILD_PATH value: "/var/lib/buildkite-agent/builds" - name: CONDUIT_TOKEN @@ -78,5 +78,5 @@ spec: - name: workdir emptyDir: {} nodeSelector: - cloud.google.com/gke-nodepool: linux-32 + cloud.google.com/gke-nodepool: linux-agents-2 terminationGracePeriodSeconds: 30 diff --git a/kubernetes/cron/buildkite.yaml b/kubernetes/cron/buildkite.yaml index 3b2bf93..6ab788c 100644 --- a/kubernetes/cron/buildkite.yaml +++ b/kubernetes/cron/buildkite.yaml @@ -13,7 +13,7 @@ # limitations under the License. -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: buildkite-stats @@ -60,4 +60,4 @@ spec: value: "main" restartPolicy: Never nodeSelector: - cloud.google.com/gke-nodepool: service \ No newline at end of file + cloud.google.com/gke-nodepool: default-pool \ No newline at end of file diff --git a/scripts/metrics/download_buildkite_builds_pg.sh b/scripts/metrics/download_buildkite_builds_pg.sh new file mode 100644 index 0000000..2bed402 --- /dev/null +++ b/scripts/metrics/download_buildkite_builds_pg.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +echo "downloading buildkite builds..." +env \ No newline at end of file