revi-archive/llvm-premerge-checks
1
0
Fork 0
Code Activity

update buildkite monitoring

Browse source
This commit is contained in:
Mikhail Goncharov 2021-06-28 10:16:36 +02:00
parent 19e290d0cb
commit ff4e998ef4
1 changed files with 15 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
scripts/metrics/load_buildkite.py
View file

@ -133,14 +133,12 @@ def insert_new_builds(conn):
cnt = 0 cnt = 0
for b in all_builds: for b in all_builds:
with conn.cursor() as c: with conn.cursor() as c:
c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),)) c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s) ON CONFLICT (id) DO UPDATE SET raw = %s',
if c.fetchone()[0] == 0: [b.get('id'), b, b])
c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s) ON CONFLICT (id) DO UPDATE SET raw = %s', cnt += 1
[b.get('id'), psycopg2.extras.Json(b), psycopg2.extras.Json(b)]) if cnt % 100 == 0:
cnt += 1 logging.info(f'{cnt} builds inserted / updated')
if cnt % 100 == 0: conn.commit()
logging.info(f'{cnt} builds inserted')
conn.commit()
conn.commit() conn.commit()
logging.info(f'{cnt} builds inserted') logging.info(f'{cnt} builds inserted')
return cnt return cnt
@ -165,12 +163,13 @@ def insert_all_builds(conn):
page += 1 page += 1
for b in x: for b in x:
with conn.cursor() as c: with conn.cursor() as c:
c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s) ON CONFLICT (id) DO NOTHING', c.execute('SELECT count(1) FROM builds WHERE id = %s', (b.get('id'),))
[b.get('id'), b]) if c.fetchone()[0] == 0:
cnt += 1 c.execute('INSERT INTO builds (id, raw) VALUES (%s, %s)', [b.get('id'), b])
if cnt % 100 == 0: cnt += 1
logging.info(f'{cnt} builds inserted') if cnt % 100 == 0:
conn.commit() logging.info(f'{cnt} builds inserted')
conn.commit()
conn.commit() conn.commit()
logging.info(f'{cnt} builds inserted') logging.info(f'{cnt} builds inserted')
return cnt return cnt
@ -284,9 +283,9 @@ if __name__ == '__main__':
logging.basicConfig(level='INFO', format='%(levelname)-7s %(message)s') logging.basicConfig(level='INFO', format='%(levelname)-7s %(message)s')
cn = connect() cn = connect()
logging.info('downloading buildkite data') logging.info('downloading buildkite data')
# insert_all_builds(cn) #insert_all_builds(cn)
insert_new_builds(cn) insert_new_builds(cn)
# update_running_builds(cn) update_running_builds(cn)
insert_new_jobs(cn) insert_new_jobs(cn)
download_job_artifacts_list(cn) download_job_artifacts_list(cn)
download_job_artifacts(cn) download_job_artifacts(cn)