Improve Diviner handling of paths and remarkup

Summary:
  - Currently, the atomizers don't emit atoms with the right file in all cases. Make them always emit it correctly.
  - Currently, we use absolute paths in some cases and relative paths in other cases. Use them consistently: relative when storing/presenting, absolute when accessing data.
  - Don't preserve linebreaks when marking up documentation (documentation is generally wrapped at 80col, but should not be wrapped in this way when displayed).
  - Markup Diviner link rules (albeit uselesly).

Test Plan:
Before:

{F33044}

After:

{F33045}

Reviewers: chad

Reviewed By: chad

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D4992

src/__phutil_library_map__.php

@@ -474,6 +474,7 @@ phutil_register_library_map(array(
     'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
     'DivinerPublishCache' => 'applications/diviner/cache/DivinerPublishCache.php',
     'DivinerPublisher' => 'applications/diviner/publisher/DivinerPublisher.php',
+    'DivinerRemarkupRuleSymbol' => 'applications/diviner/markup/DivinerRemarkupRuleSymbol.php',
     'DivinerRenderer' => 'applications/diviner/renderer/DivinerRenderer.php',
     'DivinerStaticPublisher' => 'applications/diviner/publisher/DivinerStaticPublisher.php',
     'DivinerWorkflow' => 'applications/diviner/workflow/DivinerWorkflow.php',
@@ -1974,6 +1975,7 @@ phutil_register_library_map(array(
     'DivinerGenerateWorkflow' => 'DivinerWorkflow',
     'DivinerListController' => 'PhabricatorController',
     'DivinerPublishCache' => 'DivinerDiskCache',
+    'DivinerRemarkupRuleSymbol' => 'PhutilRemarkupRule',
     'DivinerStaticPublisher' => 'DivinerPublisher',
     'DivinerWorkflow' => 'PhutilArgumentWorkflow',
     'DrydockAllocatorWorker' => 'PhabricatorWorker',

src/applications/diviner/atomizer/DivinerArticleAtomizer.php

@@ -2,7 +2,7 @@
 
 final class DivinerArticleAtomizer extends DivinerAtomizer {
 
-  public function atomize($file_name, $file_data) {
+  protected function executeAtomize($file_name, $file_data) {
     $atom = $this->newAtom(DivinerAtom::TYPE_ARTICLE)
       ->setLine(1)
       ->setLength(count(explode("\n", $file_data)))

src/applications/diviner/atomizer/DivinerAtomizer.php

@@ -6,6 +6,7 @@
 abstract class DivinerAtomizer {
 
   private $book;
+  private $fileName;
 
   /**
    * If you make a significant change to an atomizer, you can bump this
@@ -15,7 +16,12 @@ abstract class DivinerAtomizer {
     return 1;
   }
 
-  abstract public function atomize($file_name, $file_data);
+  final public function atomize($file_name, $file_data) {
+    $this->fileName = $file_name;
+    return $this->executeAtomize($file_name, $file_data);
+  }
+
+  abstract protected function executeAtomize($file_name, $file_data);
 
   final public function setBook($book) {
     $this->book = $book;
@@ -29,6 +35,7 @@ abstract class DivinerAtomizer {
   protected function newAtom($type) {
     return id(new DivinerAtom())
       ->setBook($this->getBook())
+      ->setFile($this->fileName)
       ->setType($type);
   }
 

src/applications/diviner/atomizer/DivinerFileAtomizer.php

@@ -2,7 +2,7 @@
 
 final class DivinerFileAtomizer extends DivinerAtomizer {
 
-  public function atomize($file_name, $file_data) {
+  protected function executeAtomize($file_name, $file_data) {
     $atom = $this->newAtom(DivinerAtom::TYPE_FILE)
       ->setName($file_name)
       ->setFile($file_name)

src/applications/diviner/markup/DivinerRemarkupRuleSymbol.php

@@ -0,0 +1,36 @@
+<?php
+
+final class DivinerRemarkupRuleSymbol extends PhutilRemarkupRule {
+
+  public function apply($text) {
+    return $this->replaceHTML(
+      '/(?:^|\B)@{(?:(?P<type>[^:]+?):)?(?P<name>[^}]+?)}/',
+      array($this, 'markupSymbol'),
+      $text);
+  }
+
+  public function markupSymbol($matches) {
+    $type = $matches['type'];
+    $name = $matches['name'];
+
+    // Collapse sequences of whitespace into a single space.
+    $name = preg_replace('/\s+/', ' ', $name);
+
+    $book = null;
+    if (strpos($type, '@') !== false) {
+      list($type, $book) = explode('@', $type, 2);
+    }
+
+    // TODO: This doesn't actually do anything useful yet.
+
+    $link = phutil_tag(
+      'a',
+      array(
+        'href' => '#',
+      ),
+      $name);
+
+    return $this->getEngine()->storeText($link);
+  }
+
+}

src/applications/diviner/renderer/DivinerDefaultRenderer.php

@@ -172,7 +172,10 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
   }
 
   protected function getBlockMarkupEngine() {
-    return PhabricatorMarkupEngine::newMarkupEngine(array());
+    return PhabricatorMarkupEngine::newMarkupEngine(
+      array(
+        'preserve-linebreaks' => false,
+      ));
   }
 
   protected function getInlineMarkupEngine() {

src/applications/diviner/workflow/DivinerAtomizeWorkflow.php

@@ -59,9 +59,14 @@ final class DivinerAtomizeWorkflow extends DivinerWorkflow {
 
     $file_atomizer = new DivinerFileAtomizer();
 
+    foreach (array($atomizer, $file_atomizer) as $configure) {
+      $configure->setBook($this->getConfig('name'));
+    }
+
     $all_atoms = array();
     foreach ($files as $file) {
-      $data = Filesystem::readFile($file);
+      $abs_path = Filesystem::resolvePath($file, $this->getConfig('root'));
+      $data = Filesystem::readFile($abs_path);
 
       if (!$this->shouldAtomizeFile($file, $data)) {
         $console->writeLog("Skipping %s...\n", $file);
@@ -89,10 +94,6 @@ final class DivinerAtomizeWorkflow extends DivinerWorkflow {
 
     $all_atoms = array_mergev($all_atoms);
 
-    foreach ($all_atoms as $atom) {
-      $atom->setBook($this->getConfig('name'));
-    }
-
     $all_atoms = mpull($all_atoms, 'toDictionary');
     $all_atoms = ipull($all_atoms, null, 'hash');
 

src/applications/diviner/workflow/DivinerGenerateWorkflow.php

@@ -198,7 +198,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
 
 
   private function findFilesInProject() {
-    $file_hashes = id(new FileFinder($this->getConfig('root')))
+    $raw_hashes = id(new FileFinder($this->getConfig('root')))
       ->excludePath('*/.*')
       ->withType('f')
       ->setGenerateChecksums(true)
@@ -206,11 +206,13 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
 
     $version = $this->getDivinerAtomWorldVersion();
 
-    foreach ($file_hashes as $file => $md5_hash) {
+    $file_hashes = array();
+    foreach ($raw_hashes as $file => $md5_hash) {
+      $rel_file = Filesystem::readablePath($file, $this->getConfig('root'));
       // We want the hash to change if the file moves or Diviner gets updated,
       // not just if the file content changes. Derive a hash from everything
       // we care about.
-      $file_hashes[$file] = md5("{$file}\0{$md5_hash}\0{$version}").'F';
+      $file_hashes[$rel_file] = md5("{$rel_file}\0{$md5_hash}\0{$version}").'F';
     }
 
     return $file_hashes;

src/infrastructure/markup/PhabricatorMarkupEngine.php

@@ -355,6 +355,7 @@ final class PhabricatorMarkupEngine {
         'uri.allowed-protocols'),
       'syntax-highlighter.engine' => PhabricatorEnv::getEnvConfig(
         'syntax-highlighter.engine'),
+      'preserve-linebreaks' => true,
     );
   }
 
@@ -368,7 +369,7 @@ final class PhabricatorMarkupEngine {
 
     $engine = new PhutilRemarkupEngine();
 
-    $engine->setConfig('preserve-linebreaks', true);
+    $engine->setConfig('preserve-linebreaks', $options['preserve-linebreaks']);
     $engine->setConfig('pygments.enabled', $options['pygments']);
     $engine->setConfig(
       'uri.allowed-protocols',
@@ -422,6 +423,8 @@ final class PhabricatorMarkupEngine {
       $rules[] = new PhabricatorRemarkupRuleMeme();
     }
 
+    $rules[] = new DivinerRemarkupRuleSymbol();
+
     $rules[] = new PhabricatorRemarkupRuleMention();
 
     $rules[] = new PhutilRemarkupRuleBold();