Implement "words of power" against ApplicationTransactions

Summary:
Ref T2222. Differential has certain "words of power" (like `Ref T123` or `Depends on D345`) which should expand into a separate transaction when they appear anywhere in text.

Currently, they're respected in only some fields. I'm expanding them to work in any remarkup field, including comments and inline comments.

This partially generalizes transaction expansion/extraction in comments. Eventually, I'll probably implement some very soft sort of reference edge for T4036, maybe.

Test Plan: {F119368}

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T2222

Differential Revision: https://secure.phabricator.com/D8369

src/applications/differential/customfield/DifferentialRevertPlanField.php

@@ -126,4 +126,9 @@ final class DifferentialRevertPlanField
       $xaction->getNewValue());
   }
 
+  public function getApplicationTransactionRemarkupBlocks(
+    PhabricatorApplicationTransaction $xaction) {
+    return array($xaction->getNewValue());
+  }
+
 }

src/applications/differential/customfield/DifferentialSummaryField.php

@@ -118,4 +118,9 @@ final class DifferentialSummaryField
       $this->getViewer());
   }
 
+  public function getApplicationTransactionRemarkupBlocks(
+    PhabricatorApplicationTransaction $xaction) {
+    return array($xaction->getNewValue());
+  }
+
 }

src/applications/differential/customfield/DifferentialTestPlanField.php

@@ -133,4 +133,9 @@ final class DifferentialTestPlanField
       $this->getViewer());
   }
 
+  public function getApplicationTransactionRemarkupBlocks(
+    PhabricatorApplicationTransaction $xaction) {
+    return array($xaction->getNewValue());
+  }
+
 }

src/applications/differential/editor/DifferentialTransactionEditor.php

@@ -706,10 +706,11 @@ final class DifferentialTransactionEditor
   }
 
   protected function sortTransactions(array $xactions) {
+    $xactions = parent::sortTransactions();
+
     $head = array();
     $tail = array();
 
-    // Move bare comments to the end, so the actions precede them.
     foreach ($xactions as $xaction) {
       $type = $xaction->getTransactionType();
       if ($type == DifferentialTransaction::TYPE_INLINE) {
@@ -823,6 +824,78 @@ final class DifferentialTransactionEditor
     return parent::extractFilePHIDsFromCustomTransaction($object, $xaction);
   }
 
+  protected function expandCustomRemarkupBlockTransactions(
+    PhabricatorLiskDAO $object,
+    array $xactions,
+    $blocks,
+    PhutilMarkupEngine $engine) {
+
+
+    $flat_blocks = array_mergev($blocks);
+    $huge_block = implode("\n\n", $flat_blocks);
+
+    $task_map = array();
+    $task_refs = id(new ManiphestCustomFieldStatusParser())
+      ->parseCorpus($huge_block);
+    foreach ($task_refs as $match) {
+      foreach ($match['monograms'] as $monogram) {
+        $task_id = (int)trim($monogram, 'tT');
+        $task_map[$task_id] = true;
+      }
+    }
+
+    $rev_map = array();
+    $rev_refs = id(new DifferentialCustomFieldDependsOnParser())
+      ->parseCorpus($huge_block);
+    foreach ($rev_refs as $match) {
+      foreach ($match['monograms'] as $monogram) {
+        $rev_id = (int)trim($monogram, 'dD');
+        $rev_map[$rev_id] = true;
+      }
+    }
+
+    $edges = array();
+
+    if ($task_map) {
+      $tasks = id(new ManiphestTaskQuery())
+        ->setViewer($this->getActor())
+        ->withIDs(array_keys($task_map))
+        ->execute();
+
+      if ($tasks) {
+        $edge_related = PhabricatorEdgeConfig::TYPE_DREV_HAS_RELATED_TASK;
+        $edges[$edge_related] = mpull($tasks, 'getPHID', 'getPHID');
+      }
+    }
+
+    if ($rev_map) {
+      $revs = id(new DifferentialRevisionQuery())
+        ->setViewer($this->getActor())
+        ->withIDs(array_keys($rev_map))
+        ->execute();
+      $rev_phids = mpull($revs, 'getPHID', 'getPHID');
+
+      // NOTE: Skip any write attempts if a user cleverly implies a revision
+      // depends upon itself.
+      unset($rev_phids[$object->getPHID()]);
+
+      if ($revs) {
+        $edge_depends = PhabricatorEdgeConfig::TYPE_DREV_DEPENDS_ON_DREV;
+        $edges[$edge_depends] = $rev_phids;
+      }
+    }
+
+    $result = array();
+    foreach ($edges as $type => $specs) {
+      $result[] = id(new DifferentialTransaction())
+        ->setTransactionType(PhabricatorTransactions::TYPE_EDGE)
+        ->setMetadataValue('edge:type', $type)
+        ->setNewValue(array('+' => $specs));
+    }
+
+    return $result;
+  }
+
   private function renderInlineCommentsForMail(
     PhabricatorLiskDAO $object,
     array $inlines) {

src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php

@@ -435,16 +435,15 @@ abstract class PhabricatorApplicationTransactionEditor
 
     $actor = $this->requireActor();
 
-    $this->loadSubscribers($object);
-
-    $xactions = $this->applyImplicitCC($object, $xactions);
-
-    $mention_xaction = $this->buildMentionTransaction($object, $xactions);
-    if ($mention_xaction) {
-      $xactions[] = $mention_xaction;
+    // NOTE: Some transaction expansion requires that the edited object be
+    // attached.
+    foreach ($xactions as $xaction) {
+      $xaction->attachObject($object);
+      $xaction->attachViewer($actor);
     }
 
     $xactions = $this->expandTransactions($object, $xactions);
+    $xactions = $this->expandSupportTransactions($object, $xactions);
     $xactions = $this->combineTransactions($xactions);
 
     foreach ($xactions as $xaction) {
@@ -802,18 +801,14 @@ abstract class PhabricatorApplicationTransactionEditor
 
   private function buildMentionTransaction(
     PhabricatorLiskDAO $object,
-    array $xactions) {
+    array $xactions,
+    array $blocks) {
 
     if (!($object instanceof PhabricatorSubscribableInterface)) {
       return null;
     }
 
-    $texts = array();
-    foreach ($xactions as $xaction) {
-      $texts[] = $this->getRemarkupBlocksFromTransaction($xaction);
-    }
-    $texts = array_mergev($texts);
-
+    $texts = array_mergev($blocks);
     $phids = PhabricatorMarkupEngine::extractPHIDsFromMentions($texts);
 
     $this->mentionedPHIDs = $phids;
@@ -845,11 +840,7 @@ abstract class PhabricatorApplicationTransactionEditor
 
   protected function getRemarkupBlocksFromTransaction(
     PhabricatorApplicationTransaction $transaction) {
-    $texts = array();
-    if ($transaction->getComment()) {
-      $texts[] = $transaction->getComment()->getContent();
-    }
-    return $texts;
+    return $transaction->getRemarkupBlocks();
   }
 
   protected function mergeTransactions(
@@ -899,6 +890,64 @@ abstract class PhabricatorApplicationTransactionEditor
     return array($xaction);
   }
 
+
+  private function expandSupportTransactions(
+    PhabricatorLiskDAO $object,
+    array $xactions) {
+    $this->loadSubscribers($object);
+
+    $xactions = $this->applyImplicitCC($object, $xactions);
+
+    $blocks = array();
+    foreach ($xactions as $key => $xaction) {
+      $blocks[$key] = $this->getRemarkupBlocksFromTransaction($xaction);
+    }
+
+    $mention_xaction = $this->buildMentionTransaction(
+      $object,
+      $xactions,
+      $blocks);
+    if ($mention_xaction) {
+      $xactions[] = $mention_xaction;
+    }
+
+    // TODO: For now, this is just a placeholder.
+    $engine = PhabricatorMarkupEngine::getEngine('extract');
+
+    $block_xactions = $this->expandRemarkupBlockTransactions(
+      $object,
+      $xactions,
+      $blocks,
+      $engine);
+
+    foreach ($block_xactions as $xaction) {
+      $xactions[] = $xaction;
+    }
+
+    return $xactions;
+  }
+
+  private function expandRemarkupBlockTransactions(
+    PhabricatorLiskDAO $object,
+    array $xactions,
+    $blocks,
+    PhutilMarkupEngine $engine) {
+    return $this->expandCustomRemarkupBlockTransactions(
+      $object,
+      $xactions,
+      $blocks,
+      $engine);
+  }
+
+  protected function expandCustomRemarkupBlockTransactions(
+    PhabricatorLiskDAO $object,
+    array $xactions,
+    $blocks,
+    PhutilMarkupEngine $engine) {
+    return array();
+  }
+
+
   /**
    * Attempt to combine similar transactions into a smaller number of total
    * transactions. For example, two transactions which edit the title of an
@@ -1830,7 +1879,6 @@ abstract class PhabricatorApplicationTransactionEditor
     }
     $blocks = array_mergev($blocks);
 
-
     $phids = array();
     if ($blocks) {
       $phids[] = PhabricatorMarkupEngine::extractFilePHIDsFromEmbeddedFiles(

src/applications/transactions/storage/PhabricatorApplicationTransaction.php

@@ -138,6 +138,30 @@ abstract class PhabricatorApplicationTransaction
     return $this->assertAttached($this->object);
   }
 
+  public function getRemarkupBlocks() {
+    $blocks = array();
+
+    switch ($this->getTransactionType()) {
+      case PhabricatorTransactions::TYPE_CUSTOMFIELD:
+        $field = $this->getTransactionCustomField();
+        if ($field) {
+          $custom_blocks = $field->getApplicationTransactionRemarkupBlocks(
+            $this);
+          foreach ($custom_blocks as $custom_block) {
+            $blocks[] = $custom_block;
+          }
+        }
+        break;
+    }
+
+    if ($this->getComment()) {
+      $blocks[] = $this->getComment()->getContent();
+    }
+
+    return $blocks;
+  }
+
+
 /* -(  Rendering  )---------------------------------------------------------- */
 
   public function setRenderingTarget($rendering_target) {

src/infrastructure/customfield/field/PhabricatorCustomField.php

@@ -848,6 +848,18 @@ abstract class PhabricatorCustomField {
   }
 
 
+  /**
+   * @task appxaction
+   */
+  public function getApplicationTransactionRemarkupBlocks(
+    PhabricatorApplicationTransaction $xaction) {
+    if ($this->proxy) {
+      return $this->proxy->getApplicationTransactionRemarkupBlocks($xaction);
+    }
+    return array();
+  }
+
+
   /**
    * @task appxaction
    */

src/infrastructure/markup/PhabricatorMarkupEngine.php

@@ -379,6 +379,12 @@ final class PhabricatorMarkupEngine {
   //    $engine->setConfig('diviner.renderer', new DivinerDefaultRenderer());
         $engine->setConfig('header.generate-toc', true);
         break;
+      case 'extract':
+        // Engine used for reference/edge extraction. Turn off anything which
+        // is slow and doesn't change reference extraction.
+        $engine = self::newMarkupEngine(array());
+        $engine->setConfig('pygments.enabled', false);
+        break;
       default:
         throw new Exception("Unknown engine ruleset: {$ruleset}!");
     }