revi-archive/llvm-premerge-checks
1
0
Fork 0
Code Activity

update buildkite monitoring

Browse source
This commit is contained in:
Mikhail Goncharov 2021-06-28 10:16:36 +02:00
parent 19e290d0cb
commit ff4e998ef4
1 changed files with 15 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
scripts/metrics/load_buildkite.py
View file

@ -133,13 +133,11 @@ def insert_new_builds(conn):
cnt = 0 cnt = 0
for b in all_builds: for b in all_builds:
with conn.cursor() as c: with conn.cursor() as c:
c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),))
if c.fetchone()[0] == 0:
c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s) ON CONFLICT (id) DO UPDATE SET raw = %s', c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s) ON CONFLICT (id) DO UPDATE SET raw = %s',
[b.get('id'), psycopg2.extras.Json(b), psycopg2.extras.Json(b)]) [b.get('id'), b, b])
cnt += 1 cnt += 1
if cnt % 100 == 0: if cnt % 100 == 0:
logging.info(f'{cnt} builds inserted') logging.info(f'{cnt} builds inserted / updated')
conn.commit() conn.commit()
conn.commit() conn.commit()
logging.info(f'{cnt} builds inserted') logging.info(f'{cnt} builds inserted')
@ -165,8 +163,9 @@ def insert_all_builds(conn):
page += 1 page += 1
for b in x: for b in x:
with conn.cursor() as c: with conn.cursor() as c:
c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s) ON CONFLICT (id) DO NOTHING', c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),))
[b.get('id'), b]) if c.fetchone()[0] == 0:
c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s)', [b.get('id'), b])
cnt += 1 cnt += 1
if cnt % 100 == 0: if cnt % 100 == 0:
logging.info(f'{cnt} builds inserted') logging.info(f'{cnt} builds inserted')
@ -284,9 +283,9 @@ if __name__ == '__main__':
logging.basicConfig(level='INFO', format='%(levelname)-7s %(message)s') logging.basicConfig(level='INFO', format='%(levelname)-7s %(message)s')
cn = connect() cn = connect()
logging.info('downloading buildkite data') logging.info('downloading buildkite data')
# insert_all_builds(cn) #insert_all_builds(cn)
insert_new_builds(cn) insert_new_builds(cn)
# update_running_builds(cn) update_running_builds(cn)
insert_new_jobs(cn) insert_new_jobs(cn)
download_job_artifacts_list(cn) download_job_artifacts_list(cn)
download_job_artifacts(cn) download_job_artifacts(cn)