Add a ref cache and index construction to Diviner

Summary: Cache refs in a single large index; rebuild the main index from them.

Test Plan: {F32334}

Reviewers: btrahan, vrana, chad

Reviewed By: chad

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D4900

src/applications/diviner/atom/DivinerAtom.php

@@ -284,11 +284,17 @@ final class DivinerAtom {
   }
 
   public function getRef() {
+    $group = null;
+    if ($this->docblockMeta) {
+      $group = $this->getDocblockMetaValue('group');
+    }
+
     return id(new DivinerAtomRef())
       ->setBook($this->getBook())
       ->setContext($this->getContext())
       ->setType($this->getType())
-      ->setName($this->getName());
+      ->setName($this->getName())
+      ->setGroup($group);
   }
 
   public static function newFromDictionary(array $dictionary) {

src/applications/diviner/atom/DivinerAtomRef.php

@@ -6,6 +6,39 @@ final class DivinerAtomRef {
   private $context;
   private $type;
   private $name;
+  private $group;
+  private $summary;
+  private $index;
+
+  public function getSortKey() {
+    return implode(
+      "\0",
+      array(
+        $this->getName(),
+        $this->getType(),
+        $this->getContext(),
+        $this->getBook(),
+        $this->getIndex(),
+      ));
+  }
+
+  public function setIndex($index) {
+    $this->index = $index;
+    return $this;
+  }
+
+  public function getIndex() {
+    return $this->index;
+  }
+
+  public function setSummary($summary) {
+    $this->summary = $summary;
+    return $this;
+  }
+
+  public function getSummary() {
+    return $this->summary;
+  }
 
   public function setName($name) {
     $normal_name = self::normalizeString($name);
@@ -53,17 +86,34 @@ final class DivinerAtomRef {
     return $this->book;
   }
 
+  public function setGroup($group) {
+    $this->group = $group;
+    return $this;
+  }
+
+  public function getGroup() {
+    return $this->group;
+  }
+
   public function toDictionary() {
     return array(
       'book'    => $this->getBook(),
       'context' => $this->getContext(),
       'type'    => $this->getType(),
       'name'    => $this->getName(),
+      'group'   => $this->getGroup(),
+      'index'   => $this->getIndex(),
+      'summary' => $this->getSummary(),
     );
   }
 
   public function toHash() {
     $dict = $this->toDictionary();
+
+    unset($dict['group']);
+    unset($dict['index']);
+    unset($dict['summary']);
+
     ksort($dict);
     return md5(serialize($dict)).'S';
   }
@@ -74,6 +124,9 @@ final class DivinerAtomRef {
     $obj->context = idx($dict, 'context');
     $obj->type = idx($dict, 'type');
     $obj->name = idx($dict, 'name');
+    $obj->group = idx($dict, 'group');
+    $obj->index = idx($dict, 'index');
+    $obj->summary = idx($dict, 'summary');
     return $obj;
   }
 

src/applications/diviner/cache/DivinerPublishCache.php

@@ -3,6 +3,7 @@
 final class DivinerPublishCache extends DivinerDiskCache {
 
   private $pathMap;
+  private $index;
 
   public function __construct($cache_directory) {
     return parent::__construct($cache_directory, 'diviner-publish-cache');
@@ -42,4 +43,31 @@ final class DivinerPublishCache extends DivinerDiskCache {
   }
 
 
+/* -(  Index  )-------------------------------------------------------------- */
+
+  public function getIndex() {
+    if ($this->index === null) {
+      $this->index = $this->getCache()->getKey('index', array());
+    }
+    return $this->index;
+  }
+
+  public function writeIndex() {
+    $this->getCache()->setKey('index', $this->getIndex());
+  }
+
+  public function deleteAtomFromIndex($hash) {
+    $index = $this->getIndex();
+    unset($index[$hash]);
+    $this->index = $index;
+    return $this;
+  }
+
+  public function addAtomToIndex($hash, array $data) {
+    $index = $this->getIndex();
+    $index[$hash] = $data;
+    $this->index = $index;
+    return $this;
+  }
+
 }

src/applications/diviner/publisher/DivinerStaticPublisher.php

@@ -53,14 +53,15 @@ final class DivinerStaticPublisher extends DivinerPublisher {
 
       $cache->removeAtomPathsFromCache($hash);
       $cache->deleteRenderCache($hash);
+      $cache->deleteAtomFromIndex($hash);
     }
-
-    $cache->writePathMap();
   }
 
   protected function createDocumentsByHash(array $hashes) {
     $indexes = array();
 
+    $cache = $this->getPublishCache();
+
     foreach ($hashes as $hash) {
       $atom = $this->getAtomFromGraphHash($hash);
 
@@ -76,12 +77,50 @@ final class DivinerStaticPublisher extends DivinerPublisher {
           $indexes[$index] = $atom;
           $paths[] = $index;
         }
+
+        $this->addAtomToIndex($hash, $atom);
       }
 
-      $this->getPublishCache()->addAtomPathsToCache($hash, $paths);
+      $cache->addAtomPathsToCache($hash, $paths);
     }
 
-    $this->getPublishCache()->writePathMap();
+    foreach ($indexes as $index => $atoms) {
+      // TODO: Publish disambiguation pages.
+    }
+
+    $this->publishIndex();
+
+    $cache->writePathMap();
+    $cache->writeIndex();
+  }
+
+  private function publishIndex() {
+    $index = $this->getPublishCache()->getIndex();
+    $refs = array();
+    foreach ($index as $hash => $dictionary) {
+      $refs[$hash] = DivinerAtomRef::newFromDictionary($dictionary);
+    }
+
+    $content = $this->getRenderer()->renderAtomIndex($refs);
+
+    $path = implode(
+      DIRECTORY_SEPARATOR,
+      array(
+        $this->getConfig('root'),
+        'docs',
+        $this->getConfig('name'),
+        'index.html',
+      ));
+
+    Filesystem::writeFile($path, $content);
+  }
+
+  private function addAtomToIndex($hash, DivinerAtom $atom) {
+    $ref = $atom->getRef();
+    $ref->setIndex($this->getAtomSimilarIndex($atom));
+    $ref->setSummary($this->getRenderer()->renderAtomSummary($atom));
+
+    $this->getPublishCache()->addAtomToIndex($hash, $ref->toDictionary());
   }
 
   private function writeDocument(DivinerAtom $atom, $content) {

src/applications/diviner/renderer/DivinerDefaultRenderer.php

@@ -6,4 +6,28 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
     return "ATOM: ".$atom->getType()." ".$atom->getName()."!";
   }
 
+  public function renderAtomSummary(DivinerAtom $atom) {
+    return "A lovely atom named ".$atom->getName();
+  }
+
+  public function renderAtomIndex(array $refs) {
+    $refs = msort($refs, 'getSortKey');
+
+    $groups = mgroup($refs, 'getGroup');
+
+    $out = array();
+    foreach ($groups as $group_key => $refs) {
+      $out[] = '<h1>'.$group_key.'</h1>';
+      $out[] = '<ul>';
+      foreach ($refs as $ref) {
+        $out[] = '<li>'.$ref->getName().' - '.$ref->getSummary().'</li>';
+      }
+      $out[] = '</ul>';
+    }
+    $out = implode("\n", $out);
+
+    return $out;
+  }
+
+
 }

src/applications/diviner/renderer/DivinerRenderer.php

@@ -3,5 +3,7 @@
 abstract class DivinerRenderer {
 
   abstract public function renderAtom(DivinerAtom $atom);
+  abstract public function renderAtomSummary(DivinerAtom $atom);
+  abstract public function renderAtomIndex(array $refs);
 
 }