Cron job to load BK data to DB
This commit is contained in:
Mikhail Goncharov
parent
26ffea31dd
commit
6b2003a664
7 changed files with 715 additions and 0 deletions
27
containers/stats/Dockerfile
Normal file
27
containers/stats/Dockerfile
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
FROM debian:unstable
|
||||
|
||||
RUN apt-get update ;\
|
||||
apt-get upgrade -y;\
|
||||
apt-get install -y --no-install-recommends \
|
||||
locales openssh-client gnupg ca-certificates \
|
||||
build-essential \
|
||||
zip wget git less vim \
|
||||
python3 python3-psutil python3-pip python3-setuptools pipenv \
|
||||
python3 python3-psutil python3-pip python3-setuptools pipenv \
|
||||
rsync jq tini gosu;
|
||||
|
||||
|
||||
RUN echo 'configure locale'; \
|
||||
sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
|
||||
locale-gen ;
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LANGUAGE en_US:en
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
|
||||
COPY *.sh /usr/local/bin/
|
||||
RUN chmod og+rx /usr/local/bin/*.sh
|
||||
RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/local/bin/cloud_sql_proxy;\
|
||||
chmod +x /usr/local/bin/cloud_sql_proxy
|
||||
|
||||
ENTRYPOINT ["entrypoint.sh"]
|
||||
CMD ["/bin/bash"]
|
||||
20
containers/stats/entrypoint.sh
Executable file
20
containers/stats/entrypoint.sh
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://llvm.org/LICENSE.txt
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
set -euo pipefail
|
||||
|
||||
git clone --depth 1 https://github.com/google/llvm-premerge-checks.git ~/llvm-premerge-checks
|
||||
cd ~/llvm-premerge-checks
|
||||
exec /usr/bin/tini -g -- $@
|
||||
61
kubernetes/stats-cron.yaml
Normal file
61
kubernetes/stats-cron.yaml
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://llvm.org/LICENSE.txt
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
apiVersion: batch/v1beta1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: collect-stats
|
||||
namespace: buildkite
|
||||
spec:
|
||||
schedule: "0 * * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 24
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: collect-buildkite-stats
|
||||
image: gcr.io/llvm-premerge-checks/stats:latest
|
||||
args: ["/root/llvm-premerge-checks/scripts/metrics/load_buildkite.sh"]
|
||||
env:
|
||||
- name: BUILDKITE_AGENT_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: buildkite-agent-token
|
||||
key: token
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: CONDUIT_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: conduit-api-token
|
||||
key: token
|
||||
- name: BUILDKITE_API_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: buildkite-api-token
|
||||
key: token
|
||||
- name: DB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: db-stats
|
||||
key: password
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
cloud.google.com/gke-nodepool: service
|
||||
21
scripts/metrics/Pipfile
Normal file
21
scripts/metrics/Pipfile
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
[[source]]
|
||||
url = "https://pypi.python.org/simple"
|
||||
verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
backoff = "*"
|
||||
GitPython = "*"
|
||||
lxml = "*"
|
||||
pathspec = "*"
|
||||
phabricator = "==0.8.1"
|
||||
pyaml = "*"
|
||||
requests = "*"
|
||||
python-benedict = "*"
|
||||
psycopg2-binary = "*"
|
||||
chardet = "*"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.9"
|
||||
323
scripts/metrics/Pipfile.lock
generated
Normal file
323
scripts/metrics/Pipfile.lock
generated
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "e3acba0f267d3eda606997edc26c93b7ce252026303108a9f1ae7d354c616aed"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.9"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.python.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"backoff": {
|
||||
"hashes": [
|
||||
"sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd",
|
||||
"sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.10.0"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
|
||||
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
|
||||
],
|
||||
"version": "==2020.12.5"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
|
||||
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.0.0"
|
||||
},
|
||||
"ftfy": {
|
||||
"hashes": [
|
||||
"sha256:9eb68533eb2a6124e96ed7f63049e6c519194fda3fae92629b5e0b5753cb2c8f"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==6.0.1"
|
||||
},
|
||||
"gitdb": {
|
||||
"hashes": [
|
||||
"sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0",
|
||||
"sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"
|
||||
],
|
||||
"version": "==4.0.7"
|
||||
},
|
||||
"gitpython": {
|
||||
"hashes": [
|
||||
"sha256:29fe82050709760081f588dd50ce83504feddbebdc4da6956d02351552b1c135",
|
||||
"sha256:ee24bdc93dce357630764db659edaf6b8d664d4ff5447ccfeedd2dc5c253f41e"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.1.17"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||
],
|
||||
"version": "==2.10"
|
||||
},
|
||||
"lxml": {
|
||||
"hashes": [
|
||||
"sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d",
|
||||
"sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3",
|
||||
"sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2",
|
||||
"sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae",
|
||||
"sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f",
|
||||
"sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927",
|
||||
"sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3",
|
||||
"sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7",
|
||||
"sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59",
|
||||
"sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f",
|
||||
"sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade",
|
||||
"sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96",
|
||||
"sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468",
|
||||
"sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b",
|
||||
"sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4",
|
||||
"sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354",
|
||||
"sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83",
|
||||
"sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04",
|
||||
"sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16",
|
||||
"sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791",
|
||||
"sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a",
|
||||
"sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51",
|
||||
"sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1",
|
||||
"sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a",
|
||||
"sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f",
|
||||
"sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee",
|
||||
"sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec",
|
||||
"sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969",
|
||||
"sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28",
|
||||
"sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a",
|
||||
"sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa",
|
||||
"sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106",
|
||||
"sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d",
|
||||
"sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617",
|
||||
"sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4",
|
||||
"sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92",
|
||||
"sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0",
|
||||
"sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4",
|
||||
"sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24",
|
||||
"sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2",
|
||||
"sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e",
|
||||
"sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0",
|
||||
"sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654",
|
||||
"sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2",
|
||||
"sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23",
|
||||
"sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.6.3"
|
||||
},
|
||||
"mailchecker": {
|
||||
"hashes": [
|
||||
"sha256:5b69c0053b271ee0bed6f6782111d8aace9aed151a67dcb6ff2eae30fe0d0878"
|
||||
],
|
||||
"version": "==4.0.7"
|
||||
},
|
||||
"pathspec": {
|
||||
"hashes": [
|
||||
"sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd",
|
||||
"sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.8.1"
|
||||
},
|
||||
"phabricator": {
|
||||
"hashes": [
|
||||
"sha256:a276fb41eb91b550c46612aba874160807664ff7fe430adaa0ffa319249df981",
|
||||
"sha256:fabf0f63f020256bb44bb9ea0eba82a1d9493b2c2c5cf7a37fcdab5d4f9404e2"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.8.1"
|
||||
},
|
||||
"phonenumbers": {
|
||||
"hashes": [
|
||||
"sha256:4b9d2f2165309613f32fe5057ff0604eb8e4bbb7be44f7ba77baef760d7d60e2",
|
||||
"sha256:8b0cf3df6ab75d22717af91014ca690423a85e77abc7b199748d1b3598b49a37"
|
||||
],
|
||||
"version": "==8.12.23"
|
||||
},
|
||||
"psycopg2-binary": {
|
||||
"hashes": [
|
||||
"sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c",
|
||||
"sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67",
|
||||
"sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0",
|
||||
"sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6",
|
||||
"sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db",
|
||||
"sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94",
|
||||
"sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52",
|
||||
"sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056",
|
||||
"sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b",
|
||||
"sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd",
|
||||
"sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550",
|
||||
"sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679",
|
||||
"sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83",
|
||||
"sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77",
|
||||
"sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2",
|
||||
"sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77",
|
||||
"sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2",
|
||||
"sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd",
|
||||
"sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859",
|
||||
"sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1",
|
||||
"sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25",
|
||||
"sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152",
|
||||
"sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf",
|
||||
"sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f",
|
||||
"sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729",
|
||||
"sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71",
|
||||
"sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66",
|
||||
"sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4",
|
||||
"sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449",
|
||||
"sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da",
|
||||
"sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a",
|
||||
"sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c",
|
||||
"sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb",
|
||||
"sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4",
|
||||
"sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.8.6"
|
||||
},
|
||||
"pyaml": {
|
||||
"hashes": [
|
||||
"sha256:29a5c2a68660a799103d6949167bd6c7953d031449d08802386372de1db6ad71",
|
||||
"sha256:67081749a82b72c45e5f7f812ee3a14a03b3f5c25ff36ec3b290514f8c4c4b99"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==20.4.0"
|
||||
},
|
||||
"python-benedict": {
|
||||
"hashes": [
|
||||
"sha256:1aa65ea78bd0bfd269e9289edb05d6f4e82ce62669ad0604a27d80db00a1a575",
|
||||
"sha256:af30f2a93aa15c0136c13bb528ded18b0e23171e8c567e968ee522a2b1cfa3b9"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.24.0"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
|
||||
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
|
||||
],
|
||||
"version": "==2.8.1"
|
||||
},
|
||||
"python-fsutil": {
|
||||
"hashes": [
|
||||
"sha256:02a347540d10c1616390a536ced73fd67df8d01c499f497d0ab3de3fbb236f0e",
|
||||
"sha256:dc8de800c9915e6a3333ff2e917207a1a7cc20d0e7ea0e165473fa29e16be566"
|
||||
],
|
||||
"version": "==0.5.0"
|
||||
},
|
||||
"python-slugify": {
|
||||
"hashes": [
|
||||
"sha256:6d8c5df75cd4a7c3a2d21e257633de53f52ab0265cd2d1dc62a730e8194a7380",
|
||||
"sha256:f13383a0b9fcbe649a1892b9c8eb4f8eab1d6d84b84bb7a624317afa98159cab"
|
||||
],
|
||||
"version": "==5.0.2"
|
||||
},
|
||||
"pyyaml": {
|
||||
"hashes": [
|
||||
"sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf",
|
||||
"sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696",
|
||||
"sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393",
|
||||
"sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77",
|
||||
"sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922",
|
||||
"sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5",
|
||||
"sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8",
|
||||
"sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10",
|
||||
"sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc",
|
||||
"sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018",
|
||||
"sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e",
|
||||
"sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253",
|
||||
"sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347",
|
||||
"sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183",
|
||||
"sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541",
|
||||
"sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb",
|
||||
"sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185",
|
||||
"sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc",
|
||||
"sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db",
|
||||
"sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa",
|
||||
"sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46",
|
||||
"sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122",
|
||||
"sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b",
|
||||
"sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63",
|
||||
"sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df",
|
||||
"sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc",
|
||||
"sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247",
|
||||
"sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6",
|
||||
"sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"
|
||||
],
|
||||
"version": "==5.4.1"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
|
||||
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.25.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
|
||||
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
|
||||
],
|
||||
"version": "==1.16.0"
|
||||
},
|
||||
"smmap": {
|
||||
"hashes": [
|
||||
"sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182",
|
||||
"sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2"
|
||||
],
|
||||
"version": "==4.0.0"
|
||||
},
|
||||
"text-unidecode": {
|
||||
"hashes": [
|
||||
"sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8",
|
||||
"sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"
|
||||
],
|
||||
"version": "==1.3"
|
||||
},
|
||||
"toml": {
|
||||
"hashes": [
|
||||
"sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
|
||||
"sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
|
||||
],
|
||||
"version": "==0.10.2"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
|
||||
"sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
|
||||
],
|
||||
"version": "==1.26.4"
|
||||
},
|
||||
"wcwidth": {
|
||||
"hashes": [
|
||||
"sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
|
||||
"sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
|
||||
],
|
||||
"version": "==0.2.5"
|
||||
},
|
||||
"xmltodict": {
|
||||
"hashes": [
|
||||
"sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21",
|
||||
"sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051"
|
||||
],
|
||||
"version": "==0.12.0"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
257
scripts/metrics/load_buildkite.py
Normal file
257
scripts/metrics/load_buildkite.py
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
import sys
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
import logging
|
||||
import requests
|
||||
import os
|
||||
import dateutil
|
||||
import chardet
|
||||
from benedict import benedict
|
||||
import traceback
|
||||
|
||||
psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)
|
||||
|
||||
token = f'Bearer {os.getenv("BUILDKITE_API_TOKEN")}'
|
||||
|
||||
def connect():
|
||||
return psycopg2.connect(
|
||||
f"host=127.0.0.1 sslmode=disable dbname=stats user=stats password={os.getenv('DB_PASSWORD')}")
|
||||
|
||||
|
||||
def download_text(url):
|
||||
r = requests.get(url, allow_redirects=True,
|
||||
headers={'Authorization': token})
|
||||
if r.status_code != 200:
|
||||
raise Exception(f'response status {r.status_code}')
|
||||
try:
|
||||
return r.content.decode('utf-8').replace("\x00", "\uFFFD"), 'utf-8'
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
return r.content.decode('ascii').replace("\x00", "\uFFFD"), 'ascii'
|
||||
except:
|
||||
pass
|
||||
d = chardet.detect(r.content)
|
||||
return r.content.decode(d['encoding']).replace("\x00", "\uFFFD"), d['encoding']
|
||||
|
||||
|
||||
def download_job_logs(conn):
|
||||
logging.info('downloading job logs')
|
||||
with conn.cursor() as c:
|
||||
c.execute(f"""
|
||||
select j.id, j.raw->>'raw_log_url' url
|
||||
from jobs j
|
||||
left join artifacts a on a.job_id = j.id and a.id=j.id
|
||||
where a.id IS NULL and j.raw->>'raw_log_url' IS NOT NULL
|
||||
""")
|
||||
total = c.rowcount
|
||||
logging.info(f'will download {total} logs')
|
||||
cnt = 0
|
||||
for row in c:
|
||||
cnt += 1
|
||||
job_id = row[0]
|
||||
url = row[1]
|
||||
meta = {'filename': 'stdout'}
|
||||
try:
|
||||
content, en = download_text(url)
|
||||
meta['encoding'] = en
|
||||
with conn.cursor() as i:
|
||||
i.execute('INSERT INTO artifacts (id, job_id, content, meta) VALUES (%s, %s, %s, %s)', [job_id, job_id, content, meta])
|
||||
except:
|
||||
meta['failure'] = traceback.format_exc()
|
||||
logging.error(f'download artifact failed {meta["failure"]} {url}')
|
||||
with conn.cursor() as i:
|
||||
i.execute('INSERT INTO artifacts (id, job_id, meta) VALUES (%s, %s, %s)', [job_id, job_id, meta])
|
||||
if cnt % 100 == 0:
|
||||
logging.info(f'downloaded {cnt}/{total} logs')
|
||||
conn.commit()
|
||||
logging.info(f'downloaded {cnt} logs')
|
||||
return True
|
||||
|
||||
|
||||
def download_job_artifacts(conn):
|
||||
logging.info('downloading job artifacts')
|
||||
with conn.cursor() as c:
|
||||
c.execute(f"""
|
||||
select ja.meta from
|
||||
(select j.key,j.id job_id, a->>'id' aid, a as meta from jobs j, json_array_elements(j.meta->'artifacts') as a) as ja
|
||||
left join artifacts a on a.job_id = ja.job_id and a.id=ja.aid
|
||||
where a.id IS NULL""")
|
||||
total = c.rowcount
|
||||
logging.info(f'will download {total} artifacts')
|
||||
cnt = 0
|
||||
for row in c:
|
||||
meta = benedict(row[0])
|
||||
cnt += 1
|
||||
try:
|
||||
content, en = download_text(meta.get('download_url'))
|
||||
meta['encoding'] = en
|
||||
with conn.cursor() as i:
|
||||
i.execute('INSERT INTO artifacts (id, job_id, content, meta) VALUES (%s, %s, %s, %s)',
|
||||
[meta.get('id'), meta.get('job_id'), content, meta])
|
||||
except:
|
||||
meta['failure'] = traceback.format_exc()
|
||||
logging.error(f'download artifact failed {meta["failure"]} {meta.get("download_url")}')
|
||||
with conn.cursor() as i:
|
||||
i.execute('INSERT INTO artifacts (id, job_id, meta) VALUES (%s, %s, %s)',
|
||||
[meta.get('id'), meta.get('job_id'), meta])
|
||||
if cnt % 100 == 0:
|
||||
logging.info(f'downloaded {cnt}/{total} artifacts')
|
||||
conn.commit()
|
||||
logging.info(f'downloaded {cnt} artifacts')
|
||||
return True
|
||||
|
||||
|
||||
def insert_new_builds(conn):
|
||||
logging.info('inserting new builds')
|
||||
max_pages = 2
|
||||
while max_pages < 1000:
|
||||
logging.info(f'checking page #{max_pages}')
|
||||
re = requests.get('https://api.buildkite.com/v2/organizations/llvm-project/builds',
|
||||
params={'page': max_pages},
|
||||
headers={'Authorization': token})
|
||||
if re.status_code != 200:
|
||||
logging.error(f'list builds response status: {re.status_code}')
|
||||
sys.exit(1)
|
||||
x = re.json()
|
||||
existing = 0
|
||||
new = 0
|
||||
for b in x:
|
||||
if (b['state'] == 'running') or (b['state'] == 'scheduled'):
|
||||
new += 1
|
||||
continue
|
||||
with conn.cursor() as c:
|
||||
c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),))
|
||||
if c.fetchone()[0] == 0:
|
||||
new += 1
|
||||
else:
|
||||
existing += 1
|
||||
logging.info(f'new {new} existing {existing}')
|
||||
if new == 0:
|
||||
break
|
||||
max_pages += 10
|
||||
max_pages += 5
|
||||
logging.info(f'will load {max_pages} pages')
|
||||
page = 1
|
||||
all_builds = []
|
||||
# Read #max_pages first in order to not miss any builds that are moved due to new inserts.
|
||||
while page <= max_pages:
|
||||
logging.info(f'loading page {page}')
|
||||
re = requests.get('https://api.buildkite.com/v2/organizations/llvm-project/builds',
|
||||
params={'page': page},
|
||||
headers={'Authorization': token})
|
||||
if re.status_code != 200:
|
||||
print('response status', re.status_code, re)
|
||||
break
|
||||
x = re.json()
|
||||
if x == []:
|
||||
logging.warning('empty response')
|
||||
break
|
||||
all_builds.extend(x)
|
||||
page += 1
|
||||
# Now insert new builds in reverse order so that we can resume correctly if operation has failed.
|
||||
all_builds.reverse()
|
||||
logging.info(f'{len(all_builds)} builds loaded')
|
||||
cnt = 0
|
||||
for b in all_builds:
|
||||
if (b['state'] == 'running') or (b['state'] == 'scheduled'):
|
||||
continue
|
||||
with conn.cursor() as c:
|
||||
c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),))
|
||||
if c.fetchone()[0] == 0:
|
||||
c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s)', [b.get('id'), psycopg2.extras.Json(b)])
|
||||
cnt += 1
|
||||
if cnt % 100 == 0:
|
||||
logging.info(f'{cnt} builds inserted')
|
||||
conn.commit()
|
||||
conn.commit()
|
||||
logging.info(f'{cnt} builds inserted')
|
||||
return cnt
|
||||
|
||||
|
||||
def download_job_artifacts_list(conn):
|
||||
logging.info('download jobs artifact lsits')
|
||||
with conn.cursor() as c:
|
||||
c.execute("""
|
||||
SELECT key, raw->>'artifacts_url', meta
|
||||
FROM jobs
|
||||
WHERE (meta->>'artifacts' IS NULL) AND (raw->>'artifacts_url' IS NOT NULL)""")
|
||||
cnt = 0
|
||||
total = c.rowcount
|
||||
logging.info(f'will download {total} artifact lists')
|
||||
for row in c:
|
||||
key = row[0]
|
||||
url = row[1]
|
||||
meta = row[2]
|
||||
if meta is None:
|
||||
meta = {}
|
||||
r = requests.get(url, allow_redirects=True, headers={'Authorization': token})
|
||||
if r.status_code != 200:
|
||||
logging.error(f'cannot load artifacts_url {r.status_code} {url}')
|
||||
continue
|
||||
meta['artifacts'] = r.json()
|
||||
with conn.cursor() as i:
|
||||
i.execute('UPDATE jobs SET meta = %s WHERE key = %s', (meta, key))
|
||||
cnt += 1
|
||||
if cnt % 100 == 0:
|
||||
logging.info(f'downloaded {cnt}/{total} artifact lists')
|
||||
conn.commit()
|
||||
logging.info(f'downloaded {cnt} artifact lists')
|
||||
conn.commit()
|
||||
|
||||
|
||||
def insert_new_jobs(conn):
|
||||
logging.info('inserting new jobs')
|
||||
with conn.cursor() as c:
|
||||
c.execute("""select bj.id, bj.jid, bj.job from
|
||||
(select b.id, j->>'id' jid, j as job from builds b, json_array_elements(b.raw->'jobs') as j) as bj
|
||||
left join jobs j on j.id = bj.jid
|
||||
where j.id IS NULL""")
|
||||
total = c.rowcount
|
||||
cnt = 0
|
||||
logging.info(f'will insert {total} jobs')
|
||||
for row in c:
|
||||
build_id = row[0]
|
||||
job_id = row[1]
|
||||
job = benedict(row[2])
|
||||
meta = {}
|
||||
# durations
|
||||
runnable_at = job.get('runnable_at')
|
||||
started_at = job.get('started_at')
|
||||
finished_at = job.get('finished_at')
|
||||
if (runnable_at is not None) and (started_at is not None) and (finished_at is not None):
|
||||
runnable_at = dateutil.parser.parse(runnable_at)
|
||||
started_at = dateutil.parser.parse(started_at)
|
||||
finished_at = dateutil.parser.parse(finished_at)
|
||||
meta['queue_time'] = (started_at - runnable_at).total_seconds()
|
||||
meta['run_time'] = (finished_at - started_at).total_seconds()
|
||||
meta['total_time'] = (finished_at - runnable_at).total_seconds()
|
||||
# agent data
|
||||
for e in job.get('agent.meta_data', []):
|
||||
p = e.split('=')
|
||||
if p[0] == 'queue':
|
||||
meta['agent_queue'] = p[1]
|
||||
if p[0] == 'name':
|
||||
meta['agent_name'] = p[1]
|
||||
with conn.cursor() as i:
|
||||
i.execute('INSERT INTO jobs (id, build_id, raw, meta) VALUES (%s, %s, %s, %s)',
|
||||
[job_id, build_id, job, meta])
|
||||
cnt += 1
|
||||
if cnt % 100 == 0:
|
||||
logging.info(f'inserted {cnt}/{total} jobs')
|
||||
conn.commit()
|
||||
logging.info(f'inserted {cnt} jobs')
|
||||
conn.commit()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level='INFO', format='%(levelname)-7s %(message)s')
|
||||
print(os.environ)
|
||||
cn = connect()
|
||||
logging.info('downloading buildkite data')
|
||||
insert_new_builds(cn)
|
||||
insert_new_jobs(cn)
|
||||
download_job_artifacts_list(cn)
|
||||
download_job_artifacts(cn)
|
||||
download_job_logs(cn)
|
||||
6
scripts/metrics/load_buildkite.sh
Executable file
6
scripts/metrics/load_buildkite.sh
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env bash
|
||||
echo "loading buildkite data"
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
||||
cd $SCRIPT_DIR
|
||||
pipenv install
|
||||
cloud_sql_proxy -instances=llvm-premerge-checks:us-central1:buildkite-stats=tcp:0.0.0.0:5432 & pipenv run python3 $SCRIPT_DIR/load_buildkite.py
|
||||
Loading…
Reference in a new issue