implemented Phabricator metrics

scripts/metrics/pull_phab_build_stats.py

@@ -19,18 +19,109 @@ import phabricator
 import json
 import os
 import datetime
+from typing import Dict, List, Optional
+import csv
+
+
+_PRE_MERGE_PHID = 'PHID-HMCP-bfkbtacsszhg3feydpo6'
+
+
+class PhabResponse:
+
+    def __init__(self, revision_dict: Dict):
+        self.revision_dict = revision_dict
+
+    @property
+    def id(self) -> str:
+        return self.revision_dict['id']
+
+    @property
+    def phid(self) -> str:
+        return self.revision_dict['phid']
+
+    def __str__(self):
+        return str(self.revision_dict)
+
+
+class Revision(PhabResponse):
+
+    def __init__(self, revision_dict):
+        super().__init__(revision_dict)
+        self.buildables = []  # type: List['Buildable']
+
+    @property
+    def status(self) -> str:
+        return self.revision_dict['fields']['status']['value']
+
+    @property
+    def builds(self) -> List['Build']:
+        builds = []
+        for b in self.buildables:
+            builds.extend(b.builds)
+        return builds
+
+    @property
+    def created_date(self):
+        return self.revision_dict['fields']['dateCreated']
+
+    @property
+    def was_premerge_tested(self) -> bool:
+        return any((b.was_premerge_tested for b in self.builds))
+
+
+class Buildable(PhabResponse):
+
+    def __init__(self, revision_dict):
+        super().__init__(revision_dict)
+        self.builds = []  # type: List[Build]
+        self.revision = None  # type: Optional[Revision]
+
+    @property
+    def diff_phid(self) -> str:
+        return self.revision_dict['buildablePHID']
+
+    @property
+    def revison_phid(self) -> str:
+        return self.revision_dict['containerPHID']
+
+
+class Build(PhabResponse):
+
+    def __init__(self, revision_dict):
+        super().__init__(revision_dict)
+        self.buildable = None  # type: Optional[Buildable]
+
+    @property
+    def buildable_phid(self) -> str:
+        return self.revision_dict['fields']['buildablePHID']
+
+    @property
+    def buildplan_phid(self) -> str:
+        return self.revision_dict['fields']['buildPlanPHID']
+
+    @property
+    def was_premerge_tested(self) -> bool:
+        return self.buildplan_phid == _PRE_MERGE_PHID
 
 
 class PhabBuildPuller:
 
+    _REVISION_FILE = 'tmp/revisions.json'
+    _BUILDABLE_FILE = 'tmp/buildables.json'
+    _BUILD_FILE = 'tmp/build.json'
+    _PHAB_WEEKLY_METRICS_FILE = 'tmp/phabricator_week.csv'
+
     def __init__(self):
         self.conduit_token = None
         self.host = None
         self.phab = self._create_phab()
+        self.revisions = {}  # type: Dict[str,Revision]
+        self.buildables = {}  # type: Dict[str,Buildable]
+        self.builds = {}  # type: Dict[str,Build]
 
     def _create_phab(self) -> phabricator.Phabricator:
         phab = phabricator.Phabricator(token=self.conduit_token, host=self.host)
-        phab.update_interfaces()
+        #phab.update_interfaces()
         return phab
 
     def _load_arcrc(self):
@@ -44,11 +135,23 @@ class PhabBuildPuller:
         self.host = next(iter(arcrc['hosts']))
         self.conduit_token = arcrc['hosts'][self.host]['token']
 
-    def pull(self):
+    def run(self):
-        self.get_revisions()
+        if not os.path.exists('tmp'):
-        # TODO: match diffs (from build logs) with revisions
+            os.mkdir('tmp')
+        if not os.path.isfile(self._REVISION_FILE):
+            self.get_revisions()
+        self.parse_revisions()
+        if not os.path.isfile(self._BUILDABLE_FILE):
+            self.get_buildables()
+        self.parse_buildables()
+        if not os.path.isfile(self._BUILD_FILE):
+            self.get_builds()
+        self.parse_builds()
+        self.link_objects()
+        self.compute_metrics()
 
     def get_revisions(self):
+        print('Downloading revisions starting...')
         from_date = int(datetime.date(year=2019, month=10, day=1).strftime('%s'))
         data = []
         cursor = {
@@ -62,16 +165,103 @@ class PhabBuildPuller:
             revisions = self.phab.differential.revision.search(
                 constraints=constraints, after=after)
             data.extend(revisions.response['data'])
+            print('{} revisions...'.format(len(data)))
             after = revisions.response['cursor']['after']
             if after is None:
                 break
         print('Number of revisions:', len(data))
-        if not os.path.exists('tmp'):
+        with open(self._REVISION_FILE, 'w') as json_file:
-            os.mkdir('tmp')
-        with open('tmp/revisions.json', 'w') as json_file:
             json.dump(data, json_file)
 
+    def get_buildables(self):
+        print('Downloading buildables...')
+        data = []
+        after = None
+        while True:
+            revisions = self.phab.harbormaster.querybuildables(
+                containerPHIDs=[r.phid for r in self.revisions], after=after)
+            data.extend(revisions.response['data'])
+            print('{} buildables...'.format(len(data)))
+            after = revisions.response['cursor']['after']
+            if after is None:
+                break
+        print('Number of buildables:', len(data))
+        with open(self._BUILDABLE_FILE, 'w') as json_file:
+            json.dump(data, json_file)
+
+    def get_builds(self):
+        print('Downloading builds...')
+        data = []
+        constraints = {
+            'buildables': [r.phid for r in self.buildables]
+        }
+        after = None
+        while True:
+            revisions = self.phab.harbormaster.build.search(
+                constraints=constraints, after=after)
+            data.extend(revisions.response['data'])
+            print('{} builds...'.format(len(data)))
+            after = revisions.response['cursor']['after']
+            if after is None:
+                break
+        print('Number of buildables:', len(data))
+        with open(self._BUILD_FILE, 'w') as json_file:
+            json.dump(data, json_file)
+
+    def parse_revisions(self):
+        with open(self._REVISION_FILE) as revision_file:
+            revision_dict = json.load(revision_file)
+        self.revisions = {r.phid: r for r in (Revision(x) for x in revision_dict)}
+        print('Parsed {} revisions.'.format(len(self.revisions)))
+
+    def parse_buildables(self):
+        with open(self._BUILDABLE_FILE) as buildables_file:
+            buildable_dict = json.load(buildables_file)
+        self.buildables = {b.phid: b for b in (Buildable(x) for x in buildable_dict)}
+        print('Parsed {} buildables.'.format(len(self.buildables)))
+
+    def parse_builds(self):
+        with open(self._BUILD_FILE) as build_file:
+            build_dict = json.load(build_file)
+        self.builds = {b.phid: b for b in (Build(x) for x in build_dict)}
+        print('Parsed {} builds.'.format(len(self.builds)))
+
+    def link_objects(self):
+        for build in (b for b in self.builds.values()):
+            buildable = self.buildables[build.buildable_phid]
+            build.buildable = buildable
+            buildable.builds.append(build)
+
+        for buildable in self.buildables.values():
+            revision = self.revisions[buildable.revison_phid]
+            revision.buildables.append(buildable)
+            buildable.revision = revision
+
+    def compute_metrics(self):
+        days_dict = {}
+        for revision in self.revisions.values():
+            date = datetime.date.fromtimestamp(revision.created_date)
+            week = '{}-w{}'.format(date.year, date.isocalendar()[1])
+            days_dict.setdefault(week, []).append(revision)
+
+        csv_file = open(self._PHAB_WEEKLY_METRICS_FILE, 'w')
+        fieldnames = ['week', 'num_revisions', 'num_premt_revisions', 'precentage_premt_revisions', 'num_untested']
+        writer = csv.DictWriter(csv_file, fieldnames=fieldnames, dialect=csv.excel)
+        writer.writeheader()
+        for week in sorted(days_dict.keys()):
+            revisions = days_dict[week]
+            num_revisions = len(revisions)
+            num_premt_revisions = len([r for r in revisions if r.was_premerge_tested])
+            precentage_premt_revisions = 100.0 * num_premt_revisions / num_revisions
+            writer.writerow({
+                'week': week,
+                'num_revisions': num_revisions,
+                'num_premt_revisions': num_premt_revisions,
+                'precentage_premt_revisions': precentage_premt_revisions,
+                'num_untested': num_revisions - num_premt_revisions,
+            })
+
 
 if __name__ == '__main__':
     puller = PhabBuildPuller()
-    puller.pull()
+    puller.run()