541 lines
20 KiB
Python
Executable file
541 lines
20 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# Copyright 2019 Google LLC
|
|
#
|
|
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# https://llvm.org/LICENSE.txt
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Get data on Revisions and builds from Phabricator
|
|
|
|
import phabricator
|
|
import json
|
|
import os
|
|
import datetime
|
|
from typing import Dict, List, Optional
|
|
import csv
|
|
import time
|
|
import socket
|
|
import git
|
|
import argparse
|
|
|
|
# PHIDs of build plans used for pre-merge testing
|
|
# FIXME: how do you get these?
|
|
_PRE_MERGE_PHIDs = ['PHID-HMCP-bfkbtacsszhg3feydpo6', # beta testers
|
|
'PHID-HMCP-qbviyekvgirhhhvkzpsn', # public pre-merge tests
|
|
'PHID-HMCP-p2oc4ocen3l2yzymvg2l',
|
|
]
|
|
|
|
# query all data after this date
|
|
START_DATE = datetime.date(year=2019, month=10, day=1)
|
|
|
|
class PhabResponse:
|
|
|
|
def __init__(self, revision_dict: Dict):
|
|
self.revision_dict = revision_dict
|
|
|
|
@property
|
|
def id(self) -> str:
|
|
return self.revision_dict['id']
|
|
|
|
@property
|
|
def phid(self) -> str:
|
|
return self.revision_dict['phid']
|
|
|
|
def __str__(self):
|
|
return str(self.revision_dict)
|
|
|
|
|
|
class Revision(PhabResponse):
|
|
|
|
def __init__(self, revision_dict):
|
|
super().__init__(revision_dict)
|
|
self.buildables = [] # type: List['Buildable']
|
|
self.diffs = [] # type: List['Diff']
|
|
|
|
@property
|
|
def status(self) -> str:
|
|
return self.revision_dict['fields']['status']['value']
|
|
|
|
@property
|
|
def builds(self) -> List['Build']:
|
|
builds = []
|
|
for b in self.buildables:
|
|
builds.extend(b.builds)
|
|
return builds
|
|
|
|
@property
|
|
def created_date(self):
|
|
return self.revision_dict['fields']['dateCreated']
|
|
|
|
@property
|
|
def was_premerge_tested(self) -> bool:
|
|
return any((b.was_premerge_tested for b in self.builds))
|
|
|
|
@property
|
|
def repository_phid(self) -> str:
|
|
return self.revision_dict['fields']['repositoryPHID']
|
|
|
|
@property
|
|
def diff_phid(self) -> str:
|
|
return self.revision_dict['fields']['diffPHID']
|
|
|
|
@property
|
|
def all_diffs_have_refs(self) -> bool:
|
|
return not any(not d.has_refs for d in self.diffs)
|
|
|
|
@property
|
|
def day(self) -> datetime.date:
|
|
return datetime.date.fromtimestamp(self.created_date)
|
|
|
|
@property
|
|
def week(self) -> str:
|
|
day = self.day
|
|
return'{}-w{:02d}'.format(day.year, day.isocalendar()[1])
|
|
|
|
@property
|
|
def published(self) -> bool:
|
|
return self.status == 'published'
|
|
|
|
@property
|
|
def build_status_list(self) -> List[bool]:
|
|
return [b.passed for b in self.builds if b.was_premerge_tested]
|
|
|
|
@property
|
|
def builds_finally_succeeded(self) -> bool:
|
|
"""Return true iff one of the builds failed and the last build passed."""
|
|
return self.has_failed_builds and self.build_status_list[-1]
|
|
|
|
@property
|
|
def has_failed_builds(self) -> bool:
|
|
"""Return true iff one of the builds failed."""
|
|
return False in self.build_status_list
|
|
|
|
@property
|
|
def published_failing(self) -> bool:
|
|
"""Return true iff published and the last build failed."""
|
|
return self.was_premerge_tested and self.published \
|
|
and not self.build_status_list[-1]
|
|
|
|
@property
|
|
def all_builds_passed(self) -> bool:
|
|
return self.was_premerge_tested and all(self.build_status_list)
|
|
|
|
@property
|
|
def all_builds_failed(self) -> bool:
|
|
return self.was_premerge_tested and all(not b for b in self.build_status_list)
|
|
|
|
@property
|
|
def dateModified(self) -> datetime.datetime:
|
|
return datetime.datetime.fromtimestamp(self.revision_dict['fields']['dateModified'])
|
|
|
|
|
|
class Buildable(PhabResponse):
|
|
|
|
def __init__(self, revision_dict):
|
|
super().__init__(revision_dict)
|
|
self.builds = [] # type: List[Build]
|
|
self.revision = None # type: Optional[Revision]
|
|
|
|
@property
|
|
def diff_phid(self) -> str:
|
|
return self.revision_dict['buildablePHID']
|
|
|
|
@property
|
|
def revison_phid(self) -> str:
|
|
return self.revision_dict['containerPHID']
|
|
|
|
|
|
class Build(PhabResponse):
|
|
|
|
def __init__(self, revision_dict):
|
|
super().__init__(revision_dict)
|
|
self.buildable = None # type: Optional[Buildable]
|
|
|
|
@property
|
|
def buildable_phid(self) -> str:
|
|
return self.revision_dict['fields']['buildablePHID']
|
|
|
|
@property
|
|
def buildplan_phid(self) -> str:
|
|
return self.revision_dict['fields']['buildPlanPHID']
|
|
|
|
@property
|
|
def was_premerge_tested(self) -> bool:
|
|
result = self.buildplan_phid in _PRE_MERGE_PHIDs
|
|
return result
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
"""Returns true, if the build "passed" """
|
|
return self.was_premerge_tested and self.revision_dict['fields']['buildStatus']['value'] == 'passed'
|
|
|
|
@property
|
|
def dateModified(self) ->datetime.datetime:
|
|
return datetime.datetime.fromtimestamp(self.revision_dict['fields']['dateModified'])
|
|
|
|
class Diff(PhabResponse):
|
|
|
|
def __init__(self, revision_dict):
|
|
super().__init__(revision_dict)
|
|
self.revision = None # type: Optional[Revision]
|
|
|
|
@property
|
|
def revison_phid(self) -> str:
|
|
return self.revision_dict['fields']['revisionPHID']
|
|
|
|
@property
|
|
def _refs(self) -> List:
|
|
return self.revision_dict['fields']['refs']
|
|
|
|
@property
|
|
def has_refs(self) -> bool:
|
|
return len(self._refs) > 0
|
|
|
|
@property
|
|
def base_revision(self) -> str:
|
|
for ref in self._refs:
|
|
if ref['type'] == 'base':
|
|
return ref['identifier']
|
|
return None
|
|
|
|
@property
|
|
def base_branch(self) -> str:
|
|
for ref in self._refs:
|
|
if ref['type'] == 'branch':
|
|
return ref['name']
|
|
return None
|
|
|
|
@property
|
|
def dateCreated(self) -> datetime.datetime:
|
|
return datetime.datetime.fromtimestamp(self.revision_dict['fields']['dateCreated'])
|
|
|
|
class PhabBuildPuller:
|
|
# files/folder for sotring temporary results
|
|
_TMP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'tmp'))
|
|
_REVISION_FILE = os.path.join(_TMP_DIR, 'phab-revisions.json')
|
|
_BUILDABLE_FILE = os.path.join(_TMP_DIR, 'phab-buildables.json')
|
|
_BUILD_FILE = os.path.join(_TMP_DIR, 'phab-build.json')
|
|
_DIFF_FILE = os.path.join(_TMP_DIR, 'phab-diffs.json')
|
|
_PHAB_WEEKLY_METRICS_FILE = os.path.join(_TMP_DIR, 'phabricator_{}.csv')
|
|
|
|
def __init__(self, repo_path: str):
|
|
self.conduit_token = None
|
|
self.host = None
|
|
self.phab = self._create_phab()
|
|
self._repo_path = repo_path # type: str
|
|
self.revisions = {} # type: Dict[str, Revision]
|
|
self.buildables = {} # type: Dict[str, Buildable]
|
|
self.builds = {} # type: Dict[str, Build]
|
|
self.diffs = {} # type: Dict[str, Diff]
|
|
|
|
def _create_phab(self) -> phabricator.Phabricator:
|
|
phab = phabricator.Phabricator(token=self.conduit_token, host=self.host)
|
|
phab.update_interfaces()
|
|
return phab
|
|
|
|
def _load_arcrc(self):
|
|
"""Load arc configuration from file if not set."""
|
|
if self.conduit_token is not None or self.host is not None:
|
|
return
|
|
print('Loading configuration from ~/.arcrc file')
|
|
with open(os.path.expanduser('~/.arcrc'), 'r') as arcrc_file:
|
|
arcrc = json.load(arcrc_file)
|
|
# use the first host configured in the file
|
|
self.host = next(iter(arcrc['hosts']))
|
|
self.conduit_token = arcrc['hosts'][self.host]['token']
|
|
|
|
def run(self):
|
|
if not os.path.exists(self._TMP_DIR):
|
|
os.mkdir(self._TMP_DIR)
|
|
if not os.path.isfile(self._REVISION_FILE):
|
|
self.get_revisions()
|
|
self.parse_revisions()
|
|
if not os.path.isfile(self._BUILDABLE_FILE):
|
|
self.get_buildables()
|
|
self.parse_buildables()
|
|
if not os.path.isfile(self._BUILD_FILE):
|
|
self.get_builds()
|
|
self.parse_builds()
|
|
if not os.path.isfile(self._DIFF_FILE):
|
|
self.get_diffs()
|
|
self.parse_diffs()
|
|
self.link_objects()
|
|
self.compute_metrics('day', lambda r: r.day)
|
|
self.compute_metrics('week', lambda r: r.week)
|
|
self.count_base_revisions()
|
|
self.revision_statistics()
|
|
self.match_base_revisions_with_repo(self._repo_path)
|
|
|
|
def get_revisions(self):
|
|
print('Downloading revisions starting...')
|
|
from_date = int(START_DATE.strftime('%s'))
|
|
data = []
|
|
constraints = {
|
|
'createdStart': from_date
|
|
}
|
|
# FIXME: lots of code duplication around pagination and error handling.
|
|
# find a way to separate this into a function.
|
|
after = None
|
|
while True:
|
|
revisions = self.phab.differential.revision.search(
|
|
constraints=constraints, after=after)
|
|
data.extend(revisions.response['data'])
|
|
print('{} revisions...'.format(len(data)))
|
|
after = revisions.response['cursor']['after']
|
|
if after is None:
|
|
break
|
|
print('Number of revisions:', len(data))
|
|
with open(self._REVISION_FILE, 'w') as json_file:
|
|
json.dump(data, json_file)
|
|
|
|
def get_buildables(self):
|
|
print('Downloading buildables...')
|
|
data = []
|
|
after = None
|
|
while True:
|
|
revisions = self.phab.harbormaster.querybuildables(
|
|
containerPHIDs=[r.phid for r in self.revisions.values()], after=after)
|
|
data.extend(revisions.response['data'])
|
|
print('{} buildables...'.format(len(data)))
|
|
after = revisions.response['cursor']['after']
|
|
if after is None:
|
|
break
|
|
print('Number of buildables:', len(data))
|
|
with open(self._BUILDABLE_FILE, 'w') as json_file:
|
|
json.dump(data, json_file)
|
|
|
|
def get_builds(self):
|
|
print('Downloading builds...')
|
|
data = []
|
|
constraints = {
|
|
'buildables': [r.phid for r in self.buildables.values()]
|
|
}
|
|
after = None
|
|
while True:
|
|
# retry on timeouts
|
|
fail_count = 0
|
|
while True:
|
|
try:
|
|
revisions = self.phab.harbormaster.build.search(
|
|
constraints=constraints, after=after)
|
|
except socket.timeout:
|
|
fail_count +=1
|
|
if fail_count > 5:
|
|
raise
|
|
time.sleep(10)
|
|
continue
|
|
break
|
|
data.extend(revisions.response['data'])
|
|
print('{} builds...'.format(len(data)))
|
|
after = revisions.response['cursor']['after']
|
|
if after is None:
|
|
break
|
|
print('Number of buildables:', len(data))
|
|
with open(self._BUILD_FILE, 'w') as json_file:
|
|
json.dump(data, json_file)
|
|
|
|
def get_diffs(self):
|
|
print('Downloading diffs...')
|
|
data = []
|
|
constraints = {
|
|
'revisionPHIDs': [r.phid for r in self.revisions.values()]
|
|
}
|
|
after = None
|
|
while True:
|
|
# retry on timeouts
|
|
fail_count = 0
|
|
while True:
|
|
try:
|
|
diffs = self.phab.differential.diff.search(
|
|
constraints=constraints, after=after)
|
|
except socket.timeout:
|
|
fail_count +=1
|
|
if fail_count > 5:
|
|
raise
|
|
time.sleep(10)
|
|
continue
|
|
break
|
|
data.extend(diffs.response['data'])
|
|
print('{} diffs...'.format(len(data)))
|
|
after = diffs.response['cursor']['after']
|
|
if after is None:
|
|
break
|
|
print('Number of diffs:', len(data))
|
|
with open(self._DIFF_FILE, 'w') as json_file:
|
|
json.dump(data, json_file)
|
|
|
|
def parse_revisions(self):
|
|
with open(self._REVISION_FILE) as revision_file:
|
|
revision_dict = json.load(revision_file)
|
|
self.revisions = {r.phid: r for r in (Revision(x) for x in revision_dict)}
|
|
print('Parsed {} revisions.'.format(len(self.revisions)))
|
|
|
|
def parse_buildables(self):
|
|
with open(self._BUILDABLE_FILE) as buildables_file:
|
|
buildable_dict = json.load(buildables_file)
|
|
self.buildables = {b.phid: b for b in (Buildable(x) for x in buildable_dict)}
|
|
print('Parsed {} buildables.'.format(len(self.buildables)))
|
|
|
|
def parse_builds(self):
|
|
with open(self._BUILD_FILE) as build_file:
|
|
build_dict = json.load(build_file)
|
|
self.builds = {b.phid: b for b in (Build(x) for x in build_dict)}
|
|
print('Parsed {} builds.'.format(len(self.builds)))
|
|
|
|
def parse_diffs(self):
|
|
with open(self._DIFF_FILE) as diff_file:
|
|
diff_dict = json.load(diff_file)
|
|
self.diffs = {d.phid: d for d in (Diff(x) for x in diff_dict)}
|
|
print('Parsed {} diffs.'.format(len(self.diffs)))
|
|
|
|
def link_objects(self):
|
|
for build in (b for b in self.builds.values()):
|
|
buildable = self.buildables[build.buildable_phid]
|
|
build.buildable = buildable
|
|
buildable.builds.append(build)
|
|
|
|
for buildable in self.buildables.values():
|
|
revision = self.revisions[buildable.revison_phid]
|
|
revision.buildables.append(buildable)
|
|
buildable.revision = revision
|
|
|
|
for diff in self.diffs.values():
|
|
revision = self.revisions[diff.revison_phid]
|
|
revision.diffs.append(diff)
|
|
diff.revision = revision
|
|
|
|
def compute_metrics(self, name: str, group_function):
|
|
print('Creating metrics for {}...'.format(name))
|
|
group_dict = {}
|
|
for revision in self.revisions.values():
|
|
group_dict.setdefault(group_function(revision), []).append(revision)
|
|
|
|
csv_file = open(self._PHAB_WEEKLY_METRICS_FILE.format(name), 'w')
|
|
fieldnames = [name, '# revisions', '# tested revisions', '% tested revisions', '# untested revisions',
|
|
'# revisions without builds', '% revisions without builds', '# no repository set',
|
|
'# had failed builds', '% had failed builds', '# failed first then passed',
|
|
'% failed first then passed', '# published failing', '% published failing',
|
|
'# all passed', '% all passed']
|
|
|
|
writer = csv.DictWriter(csv_file, fieldnames=fieldnames, dialect=csv.excel)
|
|
writer.writeheader()
|
|
for group in sorted(group_dict.keys()):
|
|
revisions = group_dict[group] # type: List[Revision]
|
|
num_revisions = len(revisions)
|
|
num_premt_revisions = len([r for r in revisions if r.was_premerge_tested])
|
|
precentage_premt_revisions = 100.0 * num_premt_revisions / num_revisions
|
|
num_no_build_triggered = len([r for r in revisions if len(r.builds) == 0])
|
|
percent_no_build_triggered = 100.0 * num_no_build_triggered / num_revisions
|
|
num_no_repo = len([r for r in revisions if r.repository_phid is None])
|
|
num_had_failed_builds =len([r for r in revisions if r.has_failed_builds])
|
|
num_failed_first_then_passed = len([r for r in revisions if r.builds_finally_succeeded])
|
|
num_published_failing = len([r for r in revisions if r.published_failing])
|
|
num_all_passed = len([r for r in revisions if r.all_builds_passed])
|
|
writer.writerow({
|
|
name: group,
|
|
'# revisions': num_revisions,
|
|
'# tested revisions': num_premt_revisions,
|
|
'% tested revisions': precentage_premt_revisions,
|
|
'# untested revisions': num_revisions - num_premt_revisions,
|
|
'# revisions without builds': num_no_build_triggered,
|
|
'% revisions without builds': percent_no_build_triggered,
|
|
'# no repository set': num_no_repo,
|
|
'# had failed builds': num_had_failed_builds,
|
|
'% had failed builds': 100 * num_had_failed_builds / num_revisions,
|
|
'# failed first then passed': num_failed_first_then_passed,
|
|
'% failed first then passed': 100 * num_failed_first_then_passed / num_revisions,
|
|
'# published failing': num_published_failing,
|
|
'% published failing': 100 * num_published_failing / num_revisions,
|
|
'# all passed': num_all_passed,
|
|
'% all passed': 100*num_all_passed / num_revisions,
|
|
})
|
|
|
|
def count_base_revisions(self):
|
|
base_revisions = {}
|
|
base_branches = {}
|
|
for diff in self.diffs.values():
|
|
base_revisions.setdefault(diff.base_revision, 0)
|
|
base_revisions[diff.base_revision] += 1
|
|
|
|
base_branches.setdefault(diff.base_branch, 0)
|
|
base_branches[diff.base_branch] +=1
|
|
|
|
print(f'{len(self.diffs)} diffs are using {len(base_revisions)} different git base revisions.')
|
|
print('The top 10 revisions and their usages are:')
|
|
revisions = sorted( base_revisions.items(), key=lambda x: x[1] , reverse=True)
|
|
for i in revisions[:10]:
|
|
print(f' commit {i[0]} was used {i[1]} times')
|
|
print()
|
|
print(f'{len(self.diffs)} diffs are using {len(base_branches)} different git base branches')
|
|
branches = sorted( base_branches.items(), key=lambda x: x[1] , reverse=True)
|
|
print('The top 10 branches and their usages are:')
|
|
for i in branches[:10]:
|
|
print(f' branch {i[0]} was used {i[1]} times')
|
|
print()
|
|
|
|
def match_base_revisions_with_repo(self, repo_path: str):
|
|
repo = git.Repo(repo_path)
|
|
not_found = 0
|
|
invalid_date = 0
|
|
has_base_revision = 0
|
|
for diff in self.diffs.values():
|
|
revision = diff.base_revision
|
|
if revision is None:
|
|
continue
|
|
has_base_revision += 1
|
|
try:
|
|
commit = repo.commit(revision)
|
|
except (ValueError, git.BadName):
|
|
not_found += 1
|
|
continue
|
|
commited_date = datetime.datetime.fromtimestamp(commit.committed_date)
|
|
if commited_date > diff.dateCreated:
|
|
invalid_date += 1
|
|
|
|
print()
|
|
print(f'Of the {has_base_revision} Diffs with base revision, the base revision was NOT found in the repo for {not_found} and ')
|
|
print(f'{invalid_date} base revisions were used before being available upstream.')
|
|
print(f'So {(not_found+invalid_date)/has_base_revision*100:0.2f} % of specified the base revisions were unusable.')
|
|
|
|
def revision_statistics(self):
|
|
no_builds = 0
|
|
has_failed = 0
|
|
fail_then_pass = 0
|
|
all_passed = 0
|
|
fail_last = 0
|
|
for revision in self.revisions.values():
|
|
build_status = [b.passed for b in revision.builds]
|
|
if len(revision.builds) == 0:
|
|
no_builds += 1
|
|
continue
|
|
if False in build_status:
|
|
has_failed += 1
|
|
if build_status[-1] == True:
|
|
fail_then_pass +=1
|
|
else:
|
|
all_passed += 1
|
|
if revision.published and build_status[-1] == False:
|
|
fail_last += 1
|
|
print()
|
|
print(f'Out of the {len(self.revisions)} Revisions:')
|
|
print(f' {no_builds} had no builds.')
|
|
print(f' {has_failed} had failed builds.')
|
|
print(f' {fail_then_pass} had failed builds, but the last build passed.')
|
|
print(f' {all_passed} had only successful builds.')
|
|
print(f' {fail_last} were published with a failing build.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('repo_path')
|
|
args = parser.parse_args()
|
|
puller = PhabBuildPuller(args.repo_path)
|
|
puller.run()
|