Extract count/point data from tasks in Fact engines

Summary:
Depends on D19119. Ref T13083. This is probably still very buggy, but I'm planning to build support tools to make debugging facts easier shortly.

This generates a large number of datapoints, at least, and can render some charts which aren't all completely broken in an obvious way.

Test Plan: Ran `bin/fact analyze --all`, got some charts with lines that went up and down in the web UI.

Subscribers: yelirekim

Maniphest Tasks: T13083

Differential Revision: https://secure.phabricator.com/D19120

src/__phutil_library_map__.php

@@ -2588,6 +2588,7 @@ phutil_register_library_map(array(
     'PhabricatorCoreCreateTransaction' => 'applications/transactions/xaction/PhabricatorCoreCreateTransaction.php',
     'PhabricatorCoreTransactionType' => 'applications/transactions/xaction/PhabricatorCoreTransactionType.php',
     'PhabricatorCoreVoidTransaction' => 'applications/transactions/xaction/PhabricatorCoreVoidTransaction.php',
+    'PhabricatorCountFact' => 'applications/fact/fact/PhabricatorCountFact.php',
     'PhabricatorCountdown' => 'applications/countdown/storage/PhabricatorCountdown.php',
     'PhabricatorCountdownApplication' => 'applications/countdown/application/PhabricatorCountdownApplication.php',
     'PhabricatorCountdownController' => 'applications/countdown/controller/PhabricatorCountdownController.php',
@@ -8078,6 +8079,7 @@ phutil_register_library_map(array(
     'PhabricatorCoreCreateTransaction' => 'PhabricatorCoreTransactionType',
     'PhabricatorCoreTransactionType' => 'PhabricatorModularTransactionType',
     'PhabricatorCoreVoidTransaction' => 'PhabricatorModularTransactionType',
+    'PhabricatorCountFact' => 'PhabricatorFact',
     'PhabricatorCountdown' => array(
       'PhabricatorCountdownDAO',
       'PhabricatorPolicyInterface',

src/applications/fact/controller/PhabricatorFactChartController.php

@@ -16,6 +16,9 @@ final class PhabricatorFactChartController extends PhabricatorFactController {
 
     $key_id = id(new PhabricatorFactKeyDimension())
       ->newDimensionID($fact->getKey());
+    if (!$key_id) {
+      return new Aphront404Response();
+    }
 
     $table = $fact->newDatapoint();
     $conn_r = $table->establishConnection('r');

src/applications/fact/daemon/PhabricatorFactDaemon.php

@@ -70,20 +70,28 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
     $result = null;
 
     $datapoints = array();
+    $count = 0;
     foreach ($iterator as $key => $object) {
       $phid = $object->getPHID();
       $this->log(pht('Processing %s...', $phid));
-      $datapoints[$phid] = $this->newDatapoints($object);
-      if (count($datapoints) > 1024) {
+      $object_datapoints = $this->newDatapoints($object);
+      $count += count($object_datapoints);
+
+      $datapoints[$phid] = $object_datapoints;
+
+      if ($count > 1024) {
         $this->updateDatapoints($datapoints);
         $datapoints = array();
+        $count = 0;
       }
+
       $result = $key;
     }
 
-    if ($datapoints) {
+    if ($count) {
       $this->updateDatapoints($datapoints);
       $datapoints = array();
+      $count = 0;
     }
 
     return $result;
@@ -111,7 +119,6 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
       return;
     }
 
-
     $fact_keys = array();
     $objects = array();
     foreach ($map as $phid => $facts) {
@@ -129,9 +136,9 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
     }
 
     $key_map = id(new PhabricatorFactKeyDimension())
-      ->newDimensionMap(array_keys($fact_keys));
+      ->newDimensionMap(array_keys($fact_keys), true);
     $object_map = id(new PhabricatorFactObjectDimension())
-      ->newDimensionMap(array_keys($objects));
+      ->newDimensionMap(array_keys($objects), true);
 
     $table = new PhabricatorFactIntDatapoint();
     $conn = $table->establishConnection('w');

src/applications/fact/engine/PhabricatorFactEngine.php

@@ -35,4 +35,8 @@ abstract class PhabricatorFactEngine extends Phobject {
     return $this->factMap[$key];
   }
 
+  final protected function getViewer() {
+    return PhabricatorUser::getOmnipotentUser();
+  }
+
 }

src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php

@@ -5,8 +5,77 @@ final class PhabricatorFactManiphestTaskEngine
 
   public function newFacts() {
     return array(
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.count.create'),
+
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.create'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.status'),
+
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.count.create.project'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.count.assign.project'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.create.project'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.status.project'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.assign.project'),
+
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.count.create.owner'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.count.assign.owner'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.create.owner'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.status.owner'),
+      id(new PhabricatorCountFact())
+        ->setKey('tasks.open-count.assign.owner'),
+
       id(new PhabricatorPointsFact())
-        ->setKey('tasks.count.open'),
+        ->setKey('tasks.points.create'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.points.score'),
+
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.create'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.status'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.score'),
+
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.points.create.project'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.points.assign.project'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.points.score.project'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.create.project'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.status.project'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.score.project'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.assign.project'),
+
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.points.create.owner'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.points.assign.owner'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.points.score.owner'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.create.owner'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.status.owner'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.score.project'),
+      id(new PhabricatorPointsFact())
+        ->setKey('tasks.open-points.assign.owner'),
     );
   }
 
@@ -15,18 +84,319 @@ final class PhabricatorFactManiphestTaskEngine
   }
 
   public function newDatapointsForObject(PhabricatorLiskDAO $object) {
+    $viewer = $this->getViewer();
+
+    $xaction_query = PhabricatorApplicationTransactionQuery::newQueryForObject(
+      $object);
+    $xactions = $xaction_query
+      ->setViewer($viewer)
+      ->withObjectPHIDs(array($object->getPHID()))
+      ->execute();
+
+    $xactions = msortv($xactions, 'newChronologicalSortVector');
+
+    $old_open = false;
+    $old_points = 0;
+    $old_owner = null;
+    $project_map = array();
+    $object_phid = $object->getPHID();
+    $is_create = true;
+
+    $specs = array();
     $datapoints = array();
+    foreach ($xactions as $xaction_group) {
+      $add_projects = array();
+      $rem_projects = array();
 
-    $phid = $object->getPHID();
-    $type = phid_get_type($phid);
+      $new_open = $old_open;
+      $new_points = $old_points;
+      $new_owner = $old_owner;
 
-    $datapoint = $this->getFact('tasks.count.open')
-      ->newDatapoint();
+      // TODO: Actually group concurrent transactions.
+      $xaction_group = array($xaction_group);
 
-    $datapoints[] = $datapoint
-      ->setObjectPHID($phid)
-      ->setValue(1)
-      ->setEpoch($object->getDateCreated());
+      $group_epoch = last($xaction_group)->getDateCreated();
+      foreach ($xaction_group as $xaction) {
+        $old_value = $xaction->getOldValue();
+        $new_value = $xaction->getNewValue();
+        switch ($xaction->getTransactionType()) {
+          case ManiphestTaskStatusTransaction::TRANSACTIONTYPE:
+            $new_open = !ManiphestTaskStatus::isClosedStatus($new_value);
+            break;
+          case ManiphestTaskMergedIntoTransaction::TRANSACTIONTYPE:
+            // When a task is merged into another task, it is changed to a
+            // closed status without generating a separate status transaction.
+            $new_open = false;
+            break;
+          case ManiphestTaskPointsTransaction::TRANSACTIONTYPE:
+            $new_points = (int)$xaction->getNewValue();
+            break;
+          case ManiphestTaskOwnerTransaction::TRANSACTIONTYPE:
+            $new_owner = $xaction->getNewValue();
+            break;
+          case PhabricatorTransactions::TYPE_EDGE:
+            $edge_type = $xaction->getMetadataValue('edge:type');
+            switch ($edge_type) {
+              case PhabricatorProjectObjectHasProjectEdgeType::EDGECONST:
+                $record = PhabricatorEdgeChangeRecord::newFromTransaction(
+                  $xaction);
+                $add_projects += array_fuse($record->getAddedPHIDs());
+                $rem_projects += array_fuse($record->getRemovedPHIDs());
+                break;
+            }
+            break;
+        }
+      }
+
+      // If a project was both added and removed, moot it.
+      $mix_projects = array_intersect_key($add_projects, $rem_projects);
+      $add_projects = array_diff_key($add_projects, $mix_projects);
+      $rem_projects = array_diff_key($rem_projects, $mix_projects);
+
+      $project_sets = array(
+        array(
+          'phids' => $rem_projects,
+          'scale' => -1,
+        ),
+        array(
+          'phids' => $add_projects,
+          'scale' => 1,
+        ),
+      );
+
+      if ($is_create) {
+        $action = 'create';
+        $action_points = $new_points;
+      } else {
+        $action = 'assign';
+        $action_points = $old_points;
+      }
+
+      foreach ($project_sets as $project_set) {
+        $scale = $project_set['scale'];
+        foreach ($project_set['phids'] as $project_phid) {
+          if ($old_open) {
+            $specs[] = array(
+              "tasks.open-count.{$action}.project",
+              1 * $scale,
+              $project_phid,
+            );
+
+            $specs[] = array(
+              "tasks.open-points.{$action}.project",
+              $action_points * $scale,
+              $project_phid,
+            );
+          }
+
+          $specs[] = array(
+            "tasks.count.{$action}.project",
+            1 * $scale,
+            $project_phid,
+          );
+
+          $specs[] = array(
+            "tasks.points.{$action}.project",
+            $action_points * $scale,
+            $project_phid,
+          );
+
+          if ($scale < 0) {
+            unset($project_map[$project_phid]);
+          } else {
+            $project_map[$project_phid] = $project_phid;
+          }
+        }
+      }
+
+      if ($new_owner !== $old_owner) {
+        $owner_sets = array(
+          array(
+            'phid' => $old_owner,
+            'scale' => -1,
+          ),
+          array(
+            'phid' => $new_owner,
+            'scale' => 1,
+          ),
+        );
+
+        foreach ($owner_sets as $owner_set) {
+          $owner_phid = $owner_set['phid'];
+          if ($owner_phid === null) {
+            continue;
+          }
+
+          if ($old_open) {
+            $specs[] = array(
+              "tasks.open-count.{$action}.owner",
+              1 * $scale,
+              $owner_phid,
+            );
+
+            $specs[] = array(
+              "tasks.open-points.{$action}.owner",
+              $action_points * $scale,
+              $owner_phid,
+            );
+          }
+
+          $specs[] = array(
+            "tasks.count.{$action}.owner",
+            1 * $scale,
+            $owner_phid,
+          );
+
+          $specs[] = array(
+            "tasks.points.{$action}.owner",
+            $action_points * $scale,
+            $owner_phid,
+          );
+        }
+
+        $old_owner = $new_owner;
+      }
+
+      if ($is_create) {
+        $specs[] = array(
+          'tasks.count.create',
+          1,
+        );
+        $specs[] = array(
+          'tasks.points.create',
+          $new_points,
+        );
+
+        if ($new_open) {
+          $specs[] = array(
+            'tasks.open-count.create',
+            1,
+          );
+          $specs[] = array(
+            'tasks.open-points.create',
+            $new_points,
+          );
+        }
+      } else if ($new_open !== $old_open) {
+        if ($new_open) {
+          $scale = 1;
+        } else {
+          $scale = -1;
+        }
+
+        $specs[] = array(
+          'tasks.open-count.status',
+          1 * $scale,
+        );
+
+        $specs[] = array(
+          'tasks.open-points.status',
+          $action_points * $scale,
+        );
+
+        if ($new_owner !== null) {
+          $specs[] = array(
+            'tasks.open-count.status.owner',
+            1 * $scale,
+            $new_owner,
+          );
+          $specs[] = array(
+            'tasks.open-points.status.owner',
+            $action_points * $scale,
+            $new_owner,
+          );
+        }
+
+        foreach ($project_map as $project_phid) {
+          $specs[] = array(
+            'tasks.open-count.status.project',
+            1 * $scale,
+            $project_phid,
+          );
+          $specs[] = array(
+            'tasks.open-points.status.project',
+            $action_points * $scale,
+            $new_owner,
+          );
+        }
+
+        $old_open = $new_open;
+      }
+
+      // The "score" facts only apply to rescoring tasks which already
+      // exist, so we skip them if the task is being created.
+      if (($new_points !== $old_points) && !$is_create) {
+        $delta = ($new_points - $old_points);
+
+        $specs[] = array(
+          'tasks.points.score',
+          $delta,
+        );
+
+        foreach ($project_map as $project_phid) {
+          $specs[] = array(
+            'tasks.points.score.project',
+            $delta,
+            $project_phid,
+          );
+
+          if ($old_open && $new_open) {
+            $specs[] = array(
+              'tasks.open-points.score.project',
+              $delta,
+              $project_phid,
+            );
+          }
+        }
+
+        if ($new_owner !== null) {
+          $specs[] = array(
+            'tasks.points.score.owner',
+            $delta,
+            $new_owner,
+          );
+
+          if ($old_open && $new_open) {
+            $specs[] = array(
+              'tasks.open-points.score.owner',
+              $delta,
+              $new_owner,
+            );
+          }
+        }
+
+        if ($old_open && $new_open) {
+          $specs[] = array(
+            'tasks.open-points.score',
+            $delta,
+          );
+        }
+
+        $old_points = $new_points;
+      }
+
+      foreach ($specs as $spec) {
+        $spec_key = $spec[0];
+        $spec_value = $spec[1];
+
+        $datapoint = $this->getFact($spec_key)
+          ->newDatapoint();
+
+        $datapoint
+          ->setObjectPHID($object_phid)
+          ->setValue($spec_value)
+          ->setEpoch($group_epoch);
+
+        if (isset($spec[2])) {
+          $datapoint->setDimensionPHID($spec[2]);
+        }
+
+        $datapoints[] = $datapoint;
+      }
+
+      $specs = array();
+      $is_create = false;
+    }
 
     return $datapoints;
   }

src/applications/fact/fact/PhabricatorCountFact.php

@@ -0,0 +1,9 @@
+<?php
+
+final class PhabricatorCountFact extends PhabricatorFact {
+
+  protected function newTemplateDatapoint() {
+    return new PhabricatorFactIntDatapoint();
+  }
+
+}

src/applications/fact/storage/PhabricatorFactDimension.php

@@ -6,10 +6,10 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
 
   final public function newDimensionID($key) {
     $map = $this->newDimensionMap(array($key));
-    return $map[$key];
+    return idx($map, $key);
   }
 
-  final public function newDimensionMap(array $keys) {
+  final public function newDimensionMap(array $keys, $create = false) {
     if (!$keys) {
       return array();
     }
@@ -40,6 +40,10 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
       return $map;
     }
 
+    if (!$create) {
+      return $map;
+    }
+
     $sql = array();
     foreach ($need as $key) {
       $sql[] = qsprintf(
@@ -66,7 +70,7 @@ abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
       $need);
     $rows = ipull($rows, 'id', $column);
 
-    foreach ($keys as $key) {
+    foreach ($need as $key) {
       if (isset($rows[$key])) {
         $map[$key] = (int)$rows[$key];
       } else {

src/applications/transactions/storage/PhabricatorApplicationTransaction.php

@@ -260,6 +260,11 @@ abstract class PhabricatorApplicationTransaction
     return $this->oldValueHasBeenSet;
   }
 
+  public function newChronologicalSortVector() {
+    return id(new PhutilSortVector())
+      ->addInt((int)$this->getDateCreated())
+      ->addInt((int)$this->getID());
+  }
 
 /* -(  Rendering  )---------------------------------------------------------- */