diff --git a/containers/stats/Dockerfile b/containers/stats/Dockerfile new file mode 100644 index 0000000..e8306e7 --- /dev/null +++ b/containers/stats/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:unstable + +RUN apt-get update ;\ + apt-get upgrade -y;\ + apt-get install -y --no-install-recommends \ + locales openssh-client gnupg ca-certificates \ + build-essential \ + zip wget git less vim \ + python3 python3-psutil python3-pip python3-setuptools pipenv \ + python3 python3-psutil python3-pip python3-setuptools pipenv \ + rsync jq tini gosu; + + +RUN echo 'configure locale'; \ + sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ + locale-gen ; +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +COPY *.sh /usr/local/bin/ +RUN chmod og+rx /usr/local/bin/*.sh +RUN wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -O /usr/local/bin/cloud_sql_proxy;\ + chmod +x /usr/local/bin/cloud_sql_proxy + +ENTRYPOINT ["entrypoint.sh"] +CMD ["/bin/bash"] diff --git a/containers/stats/entrypoint.sh b/containers/stats/entrypoint.sh new file mode 100755 index 0000000..442dbe6 --- /dev/null +++ b/containers/stats/entrypoint.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Copyright 2021 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -euo pipefail + +git clone --depth 1 https://github.com/google/llvm-premerge-checks.git ~/llvm-premerge-checks +cd ~/llvm-premerge-checks +exec /usr/bin/tini -g -- $@ diff --git a/kubernetes/stats-cron.yaml b/kubernetes/stats-cron.yaml new file mode 100644 index 0000000..711fd81 --- /dev/null +++ b/kubernetes/stats-cron.yaml @@ -0,0 +1,61 @@ +# Copyright 2021 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: collect-stats + namespace: buildkite +spec: + schedule: "0 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 24 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + containers: + - name: collect-buildkite-stats + image: gcr.io/llvm-premerge-checks/stats:latest + args: ["/root/llvm-premerge-checks/scripts/metrics/load_buildkite.sh"] + env: + - name: BUILDKITE_AGENT_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-agent-token + key: token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONDUIT_TOKEN + valueFrom: + secretKeyRef: + name: conduit-api-token + key: token + - name: BUILDKITE_API_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-api-token + key: token + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-stats + key: password + restartPolicy: Never + nodeSelector: + cloud.google.com/gke-nodepool: service \ No newline at end of file diff --git a/scripts/metrics/Pipfile b/scripts/metrics/Pipfile new file mode 100644 index 0000000..f4bc2f5 --- /dev/null +++ b/scripts/metrics/Pipfile @@ -0,0 +1,21 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +backoff = "*" +GitPython = "*" +lxml = "*" +pathspec = "*" +phabricator = "==0.8.1" +pyaml = "*" +requests = "*" +python-benedict = "*" +psycopg2-binary = "*" +chardet = "*" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/scripts/metrics/Pipfile.lock b/scripts/metrics/Pipfile.lock new file mode 100644 index 0000000..ff2f28f --- /dev/null +++ b/scripts/metrics/Pipfile.lock @@ -0,0 +1,323 @@ +{ + "_meta": { + "hash": { + "sha256": "e3acba0f267d3eda606997edc26c93b7ce252026303108a9f1ae7d354c616aed" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "backoff": { + "hashes": [ + "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd", + "sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4" + ], + "index": "pypi", + "version": "==1.10.0" + }, + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "index": "pypi", + "version": "==4.0.0" + }, + "ftfy": { + "hashes": [ + "sha256:9eb68533eb2a6124e96ed7f63049e6c519194fda3fae92629b5e0b5753cb2c8f" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0.1" + }, + "gitdb": { + "hashes": [ + "sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0", + "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005" + ], + "version": "==4.0.7" + }, + "gitpython": { + "hashes": [ + "sha256:29fe82050709760081f588dd50ce83504feddbebdc4da6956d02351552b1c135", + "sha256:ee24bdc93dce357630764db659edaf6b8d664d4ff5447ccfeedd2dc5c253f41e" + ], + "index": "pypi", + "version": "==3.1.17" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "version": "==2.10" + }, + "lxml": { + "hashes": [ + "sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d", + "sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3", + "sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2", + "sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae", + "sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f", + "sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927", + "sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3", + "sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7", + "sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59", + "sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f", + "sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade", + "sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96", + "sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468", + "sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b", + "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4", + "sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354", + "sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83", + "sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04", + "sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16", + "sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791", + "sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a", + "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51", + "sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1", + "sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a", + "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f", + "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee", + "sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec", + "sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969", + "sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28", + "sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a", + "sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa", + "sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106", + "sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d", + "sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617", + "sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4", + "sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92", + "sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0", + "sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4", + "sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24", + "sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2", + "sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e", + "sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0", + "sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654", + "sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2", + "sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23", + "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586" + ], + "index": "pypi", + "version": "==4.6.3" + }, + "mailchecker": { + "hashes": [ + "sha256:5b69c0053b271ee0bed6f6782111d8aace9aed151a67dcb6ff2eae30fe0d0878" + ], + "version": "==4.0.7" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "index": "pypi", + "version": "==0.8.1" + }, + "phabricator": { + "hashes": [ + "sha256:a276fb41eb91b550c46612aba874160807664ff7fe430adaa0ffa319249df981", + "sha256:fabf0f63f020256bb44bb9ea0eba82a1d9493b2c2c5cf7a37fcdab5d4f9404e2" + ], + "index": "pypi", + "version": "==0.8.1" + }, + "phonenumbers": { + "hashes": [ + "sha256:4b9d2f2165309613f32fe5057ff0604eb8e4bbb7be44f7ba77baef760d7d60e2", + "sha256:8b0cf3df6ab75d22717af91014ca690423a85e77abc7b199748d1b3598b49a37" + ], + "version": "==8.12.23" + }, + "psycopg2-binary": { + "hashes": [ + "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c", + "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67", + "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0", + "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6", + "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db", + "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94", + "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52", + "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056", + "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b", + "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd", + "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550", + "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679", + "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83", + "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77", + "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2", + "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77", + "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2", + "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd", + "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859", + "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1", + "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25", + "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152", + "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf", + "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f", + "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729", + "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71", + "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66", + "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4", + "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449", + "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da", + "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a", + "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c", + "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb", + "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4", + "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5" + ], + "index": "pypi", + "version": "==2.8.6" + }, + "pyaml": { + "hashes": [ + "sha256:29a5c2a68660a799103d6949167bd6c7953d031449d08802386372de1db6ad71", + "sha256:67081749a82b72c45e5f7f812ee3a14a03b3f5c25ff36ec3b290514f8c4c4b99" + ], + "index": "pypi", + "version": "==20.4.0" + }, + "python-benedict": { + "hashes": [ + "sha256:1aa65ea78bd0bfd269e9289edb05d6f4e82ce62669ad0604a27d80db00a1a575", + "sha256:af30f2a93aa15c0136c13bb528ded18b0e23171e8c567e968ee522a2b1cfa3b9" + ], + "index": "pypi", + "version": "==0.24.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "version": "==2.8.1" + }, + "python-fsutil": { + "hashes": [ + "sha256:02a347540d10c1616390a536ced73fd67df8d01c499f497d0ab3de3fbb236f0e", + "sha256:dc8de800c9915e6a3333ff2e917207a1a7cc20d0e7ea0e165473fa29e16be566" + ], + "version": "==0.5.0" + }, + "python-slugify": { + "hashes": [ + "sha256:6d8c5df75cd4a7c3a2d21e257633de53f52ab0265cd2d1dc62a730e8194a7380", + "sha256:f13383a0b9fcbe649a1892b9c8eb4f8eab1d6d84b84bb7a624317afa98159cab" + ], + "version": "==5.0.2" + }, + "pyyaml": { + "hashes": [ + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" + ], + "version": "==5.4.1" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "index": "pypi", + "version": "==2.25.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "version": "==1.16.0" + }, + "smmap": { + "hashes": [ + "sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182", + "sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2" + ], + "version": "==4.0.0" + }, + "text-unidecode": { + "hashes": [ + "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", + "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" + ], + "version": "==1.3" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "version": "==0.10.2" + }, + "urllib3": { + "hashes": [ + "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", + "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" + ], + "version": "==1.26.4" + }, + "wcwidth": { + "hashes": [ + "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", + "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" + ], + "version": "==0.2.5" + }, + "xmltodict": { + "hashes": [ + "sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21", + "sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051" + ], + "version": "==0.12.0" + } + }, + "develop": {} +} diff --git a/scripts/metrics/load_buildkite.py b/scripts/metrics/load_buildkite.py new file mode 100644 index 0000000..1515967 --- /dev/null +++ b/scripts/metrics/load_buildkite.py @@ -0,0 +1,257 @@ +import sys + +import psycopg2 +import psycopg2.extras +import logging +import requests +import os +import dateutil +import chardet +from benedict import benedict +import traceback + +psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) + +token = f'Bearer {os.getenv("BUILDKITE_API_TOKEN")}' + +def connect(): + return psycopg2.connect( + f"host=127.0.0.1 sslmode=disable dbname=stats user=stats password={os.getenv('DB_PASSWORD')}") + + +def download_text(url): + r = requests.get(url, allow_redirects=True, + headers={'Authorization': token}) + if r.status_code != 200: + raise Exception(f'response status {r.status_code}') + try: + return r.content.decode('utf-8').replace("\x00", "\uFFFD"), 'utf-8' + except: + pass + try: + return r.content.decode('ascii').replace("\x00", "\uFFFD"), 'ascii' + except: + pass + d = chardet.detect(r.content) + return r.content.decode(d['encoding']).replace("\x00", "\uFFFD"), d['encoding'] + + +def download_job_logs(conn): + logging.info('downloading job logs') + with conn.cursor() as c: + c.execute(f""" +select j.id, j.raw->>'raw_log_url' url +from jobs j +left join artifacts a on a.job_id = j.id and a.id=j.id +where a.id IS NULL and j.raw->>'raw_log_url' IS NOT NULL +""") + total = c.rowcount + logging.info(f'will download {total} logs') + cnt = 0 + for row in c: + cnt += 1 + job_id = row[0] + url = row[1] + meta = {'filename': 'stdout'} + try: + content, en = download_text(url) + meta['encoding'] = en + with conn.cursor() as i: + i.execute('INSERT INTO artifacts (id, job_id, content, meta) VALUES (%s, %s, %s, %s)', [job_id, job_id, content, meta]) + except: + meta['failure'] = traceback.format_exc() + logging.error(f'download artifact failed {meta["failure"]} {url}') + with conn.cursor() as i: + i.execute('INSERT INTO artifacts (id, job_id, meta) VALUES (%s, %s, %s)', [job_id, job_id, meta]) + if cnt % 100 == 0: + logging.info(f'downloaded {cnt}/{total} logs') + conn.commit() + logging.info(f'downloaded {cnt} logs') + return True + + +def download_job_artifacts(conn): + logging.info('downloading job artifacts') + with conn.cursor() as c: + c.execute(f""" +select ja.meta from +(select j.key,j.id job_id, a->>'id' aid, a as meta from jobs j, json_array_elements(j.meta->'artifacts') as a) as ja +left join artifacts a on a.job_id = ja.job_id and a.id=ja.aid +where a.id IS NULL""") + total = c.rowcount + logging.info(f'will download {total} artifacts') + cnt = 0 + for row in c: + meta = benedict(row[0]) + cnt += 1 + try: + content, en = download_text(meta.get('download_url')) + meta['encoding'] = en + with conn.cursor() as i: + i.execute('INSERT INTO artifacts (id, job_id, content, meta) VALUES (%s, %s, %s, %s)', + [meta.get('id'), meta.get('job_id'), content, meta]) + except: + meta['failure'] = traceback.format_exc() + logging.error(f'download artifact failed {meta["failure"]} {meta.get("download_url")}') + with conn.cursor() as i: + i.execute('INSERT INTO artifacts (id, job_id, meta) VALUES (%s, %s, %s)', + [meta.get('id'), meta.get('job_id'), meta]) + if cnt % 100 == 0: + logging.info(f'downloaded {cnt}/{total} artifacts') + conn.commit() + logging.info(f'downloaded {cnt} artifacts') + return True + + +def insert_new_builds(conn): + logging.info('inserting new builds') + max_pages = 2 + while max_pages < 1000: + logging.info(f'checking page #{max_pages}') + re = requests.get('https://api.buildkite.com/v2/organizations/llvm-project/builds', + params={'page': max_pages}, + headers={'Authorization': token}) + if re.status_code != 200: + logging.error(f'list builds response status: {re.status_code}') + sys.exit(1) + x = re.json() + existing = 0 + new = 0 + for b in x: + if (b['state'] == 'running') or (b['state'] == 'scheduled'): + new += 1 + continue + with conn.cursor() as c: + c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),)) + if c.fetchone()[0] == 0: + new += 1 + else: + existing += 1 + logging.info(f'new {new} existing {existing}') + if new == 0: + break + max_pages += 10 + max_pages += 5 + logging.info(f'will load {max_pages} pages') + page = 1 + all_builds = [] + # Read #max_pages first in order to not miss any builds that are moved due to new inserts. + while page <= max_pages: + logging.info(f'loading page {page}') + re = requests.get('https://api.buildkite.com/v2/organizations/llvm-project/builds', + params={'page': page}, + headers={'Authorization': token}) + if re.status_code != 200: + print('response status', re.status_code, re) + break + x = re.json() + if x == []: + logging.warning('empty response') + break + all_builds.extend(x) + page += 1 + # Now insert new builds in reverse order so that we can resume correctly if operation has failed. + all_builds.reverse() + logging.info(f'{len(all_builds)} builds loaded') + cnt = 0 + for b in all_builds: + if (b['state'] == 'running') or (b['state'] == 'scheduled'): + continue + with conn.cursor() as c: + c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),)) + if c.fetchone()[0] == 0: + c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s)', [b.get('id'), psycopg2.extras.Json(b)]) + cnt += 1 + if cnt % 100 == 0: + logging.info(f'{cnt} builds inserted') + conn.commit() + conn.commit() + logging.info(f'{cnt} builds inserted') + return cnt + + +def download_job_artifacts_list(conn): + logging.info('download jobs artifact lsits') + with conn.cursor() as c: + c.execute(""" +SELECT key, raw->>'artifacts_url', meta +FROM jobs +WHERE (meta->>'artifacts' IS NULL) AND (raw->>'artifacts_url' IS NOT NULL)""") + cnt = 0 + total = c.rowcount + logging.info(f'will download {total} artifact lists') + for row in c: + key = row[0] + url = row[1] + meta = row[2] + if meta is None: + meta = {} + r = requests.get(url, allow_redirects=True, headers={'Authorization': token}) + if r.status_code != 200: + logging.error(f'cannot load artifacts_url {r.status_code} {url}') + continue + meta['artifacts'] = r.json() + with conn.cursor() as i: + i.execute('UPDATE jobs SET meta = %s WHERE key = %s', (meta, key)) + cnt += 1 + if cnt % 100 == 0: + logging.info(f'downloaded {cnt}/{total} artifact lists') + conn.commit() + logging.info(f'downloaded {cnt} artifact lists') + conn.commit() + + +def insert_new_jobs(conn): + logging.info('inserting new jobs') + with conn.cursor() as c: + c.execute("""select bj.id, bj.jid, bj.job from +(select b.id, j->>'id' jid, j as job from builds b, json_array_elements(b.raw->'jobs') as j) as bj +left join jobs j on j.id = bj.jid +where j.id IS NULL""") + total = c.rowcount + cnt = 0 + logging.info(f'will insert {total} jobs') + for row in c: + build_id = row[0] + job_id = row[1] + job = benedict(row[2]) + meta = {} + # durations + runnable_at = job.get('runnable_at') + started_at = job.get('started_at') + finished_at = job.get('finished_at') + if (runnable_at is not None) and (started_at is not None) and (finished_at is not None): + runnable_at = dateutil.parser.parse(runnable_at) + started_at = dateutil.parser.parse(started_at) + finished_at = dateutil.parser.parse(finished_at) + meta['queue_time'] = (started_at - runnable_at).total_seconds() + meta['run_time'] = (finished_at - started_at).total_seconds() + meta['total_time'] = (finished_at - runnable_at).total_seconds() + # agent data + for e in job.get('agent.meta_data', []): + p = e.split('=') + if p[0] == 'queue': + meta['agent_queue'] = p[1] + if p[0] == 'name': + meta['agent_name'] = p[1] + with conn.cursor() as i: + i.execute('INSERT INTO jobs (id, build_id, raw, meta) VALUES (%s, %s, %s, %s)', + [job_id, build_id, job, meta]) + cnt += 1 + if cnt % 100 == 0: + logging.info(f'inserted {cnt}/{total} jobs') + conn.commit() + logging.info(f'inserted {cnt} jobs') + conn.commit() + + +if __name__ == '__main__': + logging.basicConfig(level='INFO', format='%(levelname)-7s %(message)s') + print(os.environ) + cn = connect() + logging.info('downloading buildkite data') + insert_new_builds(cn) + insert_new_jobs(cn) + download_job_artifacts_list(cn) + download_job_artifacts(cn) + download_job_logs(cn) diff --git a/scripts/metrics/load_buildkite.sh b/scripts/metrics/load_buildkite.sh new file mode 100755 index 0000000..8621cf7 --- /dev/null +++ b/scripts/metrics/load_buildkite.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +echo "loading buildkite data" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd $SCRIPT_DIR +pipenv install +cloud_sql_proxy -instances=llvm-premerge-checks:us-central1:buildkite-stats=tcp:0.0.0.0:5432 & pipenv run python3 $SCRIPT_DIR/load_buildkite.py