1
0
Fork 0
llvm-premerge-checks/scripts/metrics/repo_hist_db.py

151 lines
5.6 KiB
Python
Raw Normal View History

2021-04-27 09:56:24 +02:00
#!/usr/bin/env python3
import os
import sqlite3
import git
from repo_hist import MyCommit
import datetime
import csv
2021-04-27 09:56:24 +02:00
DB_PATH = "tmp/git_hist.sqlite"
REPO_DIR = "tmp/llvm-project"
GIT_URL = "https://github.com/llvm/llvm-project.git"
GIT_BRANCH = "main"
OUTPUT_PATH = "tmp"
2021-04-27 09:56:24 +02:00
# this was the start of using git as primary repo
MAX_AGE = datetime.datetime(year=2019, month=10, day=1, tzinfo=datetime.timezone.utc)
# Maximum age of the database before we re-create it
DB_UPDATE_INTERVAL = datetime.timedelta(days=1)
def popolate_db(
db_path: str, repo_dir: str, max_age: datetime.datetime
) -> sqlite3.Connection:
# TODO: full scan of the git history is quite slow. Maybe enable incremental
# updates. Only insert commits that are not yet in the database.
if os.path.exists(db_path):
age = datetime.datetime.now() - datetime.datetime.fromtimestamp(
os.path.getmtime(db_path)
)
if age < DB_UPDATE_INTERVAL:
print("Database is recent enough, using existing one.")
return sqlite3.connect(db_path)
os.remove(db_path)
print("Database is stale, needs updating...")
conn = sqlite3.connect(db_path)
print("Creating tables...")
create_tables(conn)
print("Scanning repository...")
parse_commits(conn, repo_dir, max_age)
print("Done populating database.")
return conn
def create_tables(conn: sqlite3.Connection):
# TODO: add more attributes as needed
# TODO: add all projects as columns
# mod_<project> column: files in the subfolder (=project) <project> were
# modified by this commit.
2021-04-27 09:56:24 +02:00
conn.execute(
""" CREATE TABLE IF NOT EXISTS commits (
hash string PRIMARY KEY,
commit_time integer,
phab_id string,
reverts_hash string,
mod_llvm boolean,
mod_clang boolean,
mod_libcxx boolean,
mod_mlir boolean
2021-04-27 09:56:24 +02:00
); """
)
conn.commit()
def parse_commits(conn: sqlite3.Connection, repo_dir: str, max_age: datetime.datetime):
if os.path.isdir(repo_dir):
print("Fetching git repo...")
repo = git.Repo(repo_dir)
repo.remotes.origin.fetch(GIT_BRANCH)
else:
print("Cloning git repo...")
git.Repo.clone_from(GIT_URL, repo_dir, bare=True)
repo = git.Repo(repo_dir)
print("repo update done.")
sql_insert_commit = """ INSERT INTO
commits (hash, commit_time, phab_id, reverts_hash)
values (?,?,?,?);
"""
sql_update_commit_project = """ UPDATE commits SET mod_{} = ? where hash = ?;"""
2021-04-27 09:56:24 +02:00
day = None
for commit in repo.iter_commits(GIT_BRANCH):
# TODO: This takes a couple of minutes, maybe try using multithreading
if commit.committed_datetime < max_age:
break
mycommit = MyCommit(commit)
if mycommit.date.day != day:
day = mycommit.date.day
print(mycommit.date)
# take a snapshot commit, nice to see progress while updating the
# database
conn.commit()
conn.execute(
sql_insert_commit,
(
mycommit.chash,
mycommit.date,
mycommit.phab_revision,
mycommit.reverts_commit_hash,
),
)
# Note: prasing the patches is quite slow
for project in mycommit.modified_projects:
# TODO find a way to make this generic for all projects, maybe user
# "ALTER TABLE" to add columns as they appear
if project in ["llvm", "libcxx", "mlir", "clang"]:
conn.execute(
sql_update_commit_project.format(project), (True, mycommit.chash)
)
2021-04-27 09:56:24 +02:00
conn.commit()
def create_csv_report(title: str, query: str, output_path: str):
2021-04-27 09:56:24 +02:00
cursor = conn.cursor()
data = cursor.execute(query)
with open(os.path.join(output_path, title + ".csv"), "w") as csv_file:
writer = csv.writer(csv_file)
# write column headers
writer.writerow([description[0] for description in cursor.description])
for row in data:
writer.writerow(row)
def run_queries(conn: sqlite3.Connection, output_path: str):
print("running queries...")
create_csv_report("full_db_dump", "select * from commits;", output_path)
query = """SELECT strftime('%Y-%m',commit_time) as month, count(hash) as num_commits, count(phab_id) as num_reviewed,
(100.0*count(phab_id)/count(hash)) as percent_reviewed, count(reverts_hash) as num_reverted,
(100.0*count(reverts_hash)/count(hash)) as percent_reverted
FROM commits
WHERE mod_{}
GROUP BY month;
"""
create_csv_report("libcxx_stats", query.format("libcxx"), output_path)
create_csv_report("mlir_stats", query.format("mlir"), output_path)
query = """SELECT strftime('%Y-%m',commit_time) as month, count(hash) as num_commits, count(phab_id) as num_reviewed,
(100.0*count(phab_id)/count(hash)) as percent_reviewed, count(reverts_hash) as num_reverted,
(100.0*count(reverts_hash)/count(hash)) as percent_reverted
FROM commits
GROUP BY month;
"""
create_csv_report("all_projects_stats", query, output_path)
2021-04-27 09:56:24 +02:00
if __name__ == "__main__":
conn = popolate_db(DB_PATH, REPO_DIR, MAX_AGE)
run_queries(conn, OUTPUT_PATH)