Add a book controller and various amenities to Diviner's live view

Summary: Ref T988. Mostly backend changes, with a very rough frontend on top of them. See Conpherence discussion.

Test Plan: {F45010}

Reviewers: btrahan, chad

Reviewed By: chad

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D6113

resources/sql/patches/20130602.morediviner.sql

@@ -0,0 +1,17 @@
+ALTER TABLE {$NAMESPACE}_diviner.diviner_livebook
+  ADD configurationData LONGTEXT COLLATE utf8_bin NOT NULL;
+
+UPDATE {$NAMESPACE}_diviner.diviner_livebook
+  SET configurationData = '{}' WHERE configurationData = '';
+
+ALTER TABLE {$NAMESPACE}_diviner.diviner_livesymbol
+  ADD title VARCHAR(255);
+
+ALTER TABLE {$NAMESPACE}_diviner.diviner_livesymbol
+  ADD groupName VARCHAR(255);
+
+ALTER TABLE {$NAMESPACE}_diviner.diviner_livesymbol
+  ADD summary LONGTEXT COLLATE utf8_bin;
+
+ALTER TABLE {$NAMESPACE}_diviner.diviner_livesymbol
+  ADD isDocumentable BOOL NOT NULL;

src/__phutil_library_map__.php

@@ -514,6 +514,7 @@ phutil_register_library_map(array(
     'DivinerAtomSearchEngine' => 'applications/diviner/query/DivinerAtomSearchEngine.php',
     'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php',
     'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php',
+    'DivinerBookController' => 'applications/diviner/controller/DivinerBookController.php',
     'DivinerBookQuery' => 'applications/diviner/query/DivinerBookQuery.php',
     'DivinerController' => 'applications/diviner/controller/DivinerController.php',
     'DivinerDAO' => 'applications/diviner/storage/DivinerDAO.php',
@@ -2342,6 +2343,7 @@ phutil_register_library_map(array(
     'DivinerAtomQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'DivinerAtomSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'DivinerAtomizeWorkflow' => 'DivinerWorkflow',
+    'DivinerBookController' => 'DivinerController',
     'DivinerBookQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'DivinerController' => 'PhabricatorController',
     'DivinerDAO' => 'PhabricatorLiskDAO',

src/applications/diviner/application/PhabricatorApplicationDiviner.php

@@ -25,6 +25,7 @@ final class PhabricatorApplicationDiviner extends PhabricatorApplication {
         'query/((?<key>[^/]+)/)?' => 'DivinerAtomListController',
       ),
       '/docs/(?P<keyword>[^/]+)/' => 'DivinerJumpController',
+      '/book/(?P<book>[^/]+)/' => 'DivinerBookController',
       '/book/'.
         '(?P<book>[^/]+)/'.
         '(?P<type>[^/]+)/'.

src/applications/diviner/controller/DivinerAtomListController.php

@@ -24,24 +24,7 @@ final class DivinerAtomListController extends DivinerController
   }
 
   public function renderResultsList(array $symbols) {
-    assert_instances_of($symbols, 'DivinerLiveSymbol');
-
-    $request = $this->getRequest();
-    $user = $request->getUser();
-
-    $list = id(new PhabricatorObjectItemListView())
-      ->setUser($user);
-
-    foreach ($symbols as $symbol) {
-      $item = id(new PhabricatorObjectItemView())
-        ->setHeader($symbol->getName())
-        ->setHref($symbol->getURI())
-        ->addIcon('none', $symbol->getType());
-
-      $list->addItem($item);
-    }
-
-    return $list;
+    return $this->renderAtomList($symbols);
   }
 
 }

src/applications/diviner/controller/DivinerBookController.php

@@ -0,0 +1,112 @@
+<?php
+
+final class DivinerBookController extends DivinerController {
+
+  private $bookName;
+
+  public function shouldAllowPublic() {
+    return true;
+  }
+
+  public function willProcessRequest(array $data) {
+    $this->bookName = $data['book'];
+  }
+
+  public function processRequest() {
+    $request = $this->getRequest();
+    $viewer = $request->getUser();
+
+    $book = id(new DivinerBookQuery())
+      ->setViewer($viewer)
+      ->withNames(array($this->bookName))
+      ->executeOne();
+
+    if (!$book) {
+      return new Aphront404Response();
+    }
+
+    $crumbs = $this->buildApplicationCrumbs();
+
+    $crumbs->addCrumb(
+      id(new PhabricatorCrumbView())
+        ->setName($book->getTitle())
+        ->setHref('/book/'.$book->getName().'/'));
+
+    $header = id(new PhabricatorHeaderView())->setHeader($book->getTitle());
+    $properties = $this->buildPropertyList($book);
+
+    $atoms = id(new DivinerAtomQuery())
+      ->setViewer($viewer)
+      ->withBookPHIDs(array($book->getPHID()))
+      ->execute();
+    $atoms = msort($atoms, 'getSortKey');
+
+    $group_spec = $book->getConfig('groups');
+    if (!is_array($group_spec)) {
+      $group_spec = array();
+    }
+
+    $groups = mgroup($atoms, 'getGroupName');
+    $groups = array_select_keys($groups, array_keys($group_spec)) + $groups;
+    if (isset($groups[''])) {
+      $no_group = $groups[''];
+      unset($groups['']);
+      $groups[''] = $no_group;
+    }
+
+    $out = array();
+    foreach ($groups as $group => $atoms) {
+      $group_info = idx($group_spec, $group);
+      if (!is_array($group_info)) {
+        $group_info = array();
+      }
+
+      $group_name = idx($group_info, 'name');
+      if (!strlen($group_name)) {
+        if (strlen($group)) {
+          $group_name = $group;
+        } else {
+          $group_name = pht('Free Radicals');
+        }
+      }
+
+      $out[] = id(new PhabricatorHeaderView())
+        ->setHeader($group_name);
+      $out[] = $this->renderAtomList($atoms);
+    }
+
+    return $this->buildApplicationPage(
+      array(
+        $crumbs,
+        $header,
+        $properties,
+        $out,
+      ),
+      array(
+        'title' => $book->getTitle(),
+        'dust' => true,
+        'device' => true,
+      ));
+  }
+
+  private function buildPropertyList(DivinerLiveBook $book) {
+    $user = $this->getRequest()->getUser();
+    $view = id(new PhabricatorPropertyListView())
+      ->setUser($user);
+
+    $policies = PhabricatorPolicyQuery::renderPolicyDescriptions(
+      $user,
+      $book);
+
+    $view->addProperty(
+      pht('Visible To'),
+      $policies[PhabricatorPolicyCapability::CAN_VIEW]);
+
+    $view->addProperty(
+      pht('Updated'),
+      phabricator_datetime($book->getDateModified(), $user));
+
+    return $view;
+  }
+
+}

src/applications/diviner/controller/DivinerController.php

@@ -21,4 +21,27 @@ abstract class DivinerController extends PhabricatorController {
     return $menu;
   }
 
+  protected function renderAtomList(array $symbols) {
+    assert_instances_of($symbols, 'DivinerLiveSymbol');
+
+    $request = $this->getRequest();
+    $user = $request->getUser();
+
+    $list = id(new PhabricatorObjectItemListView())
+      ->setUser($user);
+
+    foreach ($symbols as $symbol) {
+      $item = id(new PhabricatorObjectItemView())
+        ->setHeader($symbol->getTitle())
+        ->setHref($symbol->getURI())
+        ->addIcon('none', $symbol->getType());
+
+      $item->addAttribute(phutil_safe_html($symbol->getSummary()));
+
+      $list->addItem($item);
+    }
+
+    return $list;
+  }
+
 }

src/applications/diviner/publisher/DivinerLivePublisher.php

@@ -18,20 +18,23 @@ final class DivinerLivePublisher extends DivinerPublisher {
           ->save();
       }
 
+      $book->setConfigurationData($this->getConfigurationData())->save();
+
       $this->book = $book;
     }
     return $this->book;
   }
 
   private function loadSymbolForAtom(DivinerAtom $atom) {
-    $symbol = id(new DivinerLiveSymbol())->loadOneWhere(
-      'bookPHID = %s AND type = %s AND name = %s AND context = %ns
-        AND atomIndex = %d',
-      $this->loadBook()->getPHID(),
-      $atom->getType(),
-      $atom->getName(),
-      $atom->getContext(),
-      $this->getAtomSimilarIndex($atom));
+    $symbol = id(new DivinerAtomQuery())
+      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->withBookPHIDs(array($this->loadBook()->getPHID()))
+      ->withTypes(array($atom->getType()))
+      ->withNames(array($atom->getName()))
+      ->withContexts(array($atom->getContext()))
+      ->withIndexes(array($this->getAtomSimilarIndex($atom)))
+      ->withIncludeUndocumentable(true)
+      ->executeOne();
 
     if ($symbol) {
       return $symbol;
@@ -97,18 +100,33 @@ final class DivinerLivePublisher extends DivinerPublisher {
   protected function createDocumentsByHash(array $hashes) {
     foreach ($hashes as $hash) {
       $atom = $this->getAtomFromGraphHash($hash);
+      $ref = $atom->getRef();
 
       $symbol = $this->loadSymbolForAtom($atom);
-      $symbol->setGraphHash($hash)->save();
 
-      if ($this->shouldGenerateDocumentForAtom($atom)) {
-        $content = $this->getRenderer()->renderAtom($atom);
+      $is_documentable = $this->shouldGenerateDocumentForAtom($atom);
+
+      $symbol
+        ->setGraphHash($hash)
+        ->setIsDocumentable((int)$is_documentable)
+        ->setTitle($ref->getTitle())
+        ->setGroupName($ref->getGroup());
+
+      if ($is_documentable) {
+        $renderer = $this->getRenderer();
+        $content = $renderer->renderAtom($atom);
 
         $storage = $this->loadAtomStorageForSymbol($symbol)
           ->setAtomData($atom->toDictionary())
           ->setContent((string)phutil_safe_html($content))
           ->save();
+
+        $summary = $renderer->renderAtomSummary($atom);
+        $summary = (string)phutil_safe_html($summary);
+        $symbol->setSummary($summary);
       }
+
+      $symbol->save();
     }
   }
 

src/applications/diviner/publisher/DivinerPublisher.php

@@ -8,6 +8,12 @@ abstract class DivinerPublisher {
   private $renderer;
   private $config;
   private $symbolReverseMap;
+  private $dropCaches;
+
+  public function setDropCaches($drop_caches) {
+    $this->dropCaches = $drop_caches;
+    return $this;
+  }
 
   public function setRenderer(DivinerRenderer $renderer) {
     $renderer->setPublisher($this);
@@ -28,6 +34,10 @@ abstract class DivinerPublisher {
     return idx($this->config, $key, $default);
   }
 
+  public function getConfigurationData() {
+    return $this->config;
+  }
+
   public function setAtomCache(DivinerAtomCache $cache) {
     $this->atomCache = $cache;
     $graph_map = $this->atomCache->getGraphMap();
@@ -109,17 +119,25 @@ abstract class DivinerPublisher {
   final public function publishAtoms(array $hashes) {
     $existing = $this->loadAllPublishedHashes();
 
-    $existing_map = array_fill_keys($existing, true);
-    $hashes_map = array_fill_keys($hashes, true);
+    if ($this->dropCaches) {
+      $deleted = $existing;
+      $created = $hashes;
+    } else {
+      $existing_map = array_fill_keys($existing, true);
+      $hashes_map = array_fill_keys($hashes, true);
 
-    $deleted = array_diff_key($existing_map, $hashes_map);
-    $created = array_diff_key($hashes_map, $existing_map);
+      $deleted = array_diff_key($existing_map, $hashes_map);
+      $created = array_diff_key($hashes_map, $existing_map);
+
+      $deleted = array_keys($deleted);
+      $created = array_keys($created);
+    }
 
     echo pht('Deleting %d documents.', count($deleted))."\n";
-    $this->deleteDocumentsByHash(array_keys($deleted));
+    $this->deleteDocumentsByHash($deleted);
 
     echo pht('Creating %d documents.', count($created))."\n";
-    $this->createDocumentsByHash(array_keys($created));
+    $this->createDocumentsByHash($created);
   }
 
   protected function shouldGenerateDocumentForAtom(DivinerAtom $atom) {

src/applications/diviner/query/DivinerAtomQuery.php

@@ -10,6 +10,7 @@ final class DivinerAtomQuery
   private $types;
   private $contexts;
   private $indexes;
+  private $includeUndocumentable;
 
   private $needAtoms;
 
@@ -53,6 +54,11 @@ final class DivinerAtomQuery
     return $this;
   }
 
+  public function withIncludeUndocumentable($include) {
+    $this->includeUndocumentable = $include;
+    return $this;
+  }
+
   protected function loadPage() {
     $table = new DivinerLiveSymbol();
     $conn_r = $table->establishConnection('r');
@@ -182,6 +188,12 @@ final class DivinerAtomQuery
         $this->indexes);
     }
 
+    if (!$this->includeUndocumentable) {
+      $where[] = qsprintf(
+        $conn_r,
+        'isDocumentable = 1');
+    }
+
     $where[] = $this->buildPagingClause($conn_r);
 
     return $this->formatWhereClause($where);

src/applications/diviner/storage/DivinerLiveBook.php

@@ -6,18 +6,35 @@ final class DivinerLiveBook extends DivinerDAO
   protected $phid;
   protected $name;
   protected $viewPolicy;
+  protected $configurationData = array();
 
   public function getConfiguration() {
     return array(
       self::CONFIG_AUX_PHID => true,
+      self::CONFIG_SERIALIZATION => array(
+        'configurationData' => self::SERIALIZATION_JSON,
+      ),
     ) + parent::getConfiguration();
   }
 
+  public function getConfig($key, $default = null) {
+    return idx($this->configurationData, $key, $default);
+  }
+
+  public function setConfig($key, $value) {
+    $this->configurationData[$key] = $value;
+    return $this;
+  }
+
   public function generatePHID() {
     return PhabricatorPHID::generateNewPHID(
       PhabricatorPHIDConstants::PHID_TYPE_BOOK);
   }
 
+  public function getTitle() {
+    return $this->getConfig('title', $this->getName());
+  }
+
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 
   public function getCapabilities() {

src/applications/diviner/storage/DivinerLiveSymbol.php

@@ -12,6 +12,11 @@ final class DivinerLiveSymbol extends DivinerDAO
   protected $graphHash;
   protected $identityHash;
 
+  protected $title;
+  protected $groupName;
+  protected $summary;
+  protected $isDocumentable = 0;
+
   private $book;
   private $content;
   private $atom;
@@ -80,6 +85,10 @@ final class DivinerLiveSymbol extends DivinerDAO
     return '/'.implode('/', $parts).'/';
   }
 
+  public function getSortKey() {
+    return $this->getTitle();
+  }
+
   public function save() {
 
     // NOTE: The identity hash is just a sanity check because the unique tuple
@@ -101,6 +110,14 @@ final class DivinerLiveSymbol extends DivinerDAO
     return parent::save();
   }
 
+  public function getTitle() {
+    $title = parent::getTitle();
+    if (!strlen($title)) {
+      $title = $this->getName();
+    }
+    return $title;
+  }
+
 
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 

src/applications/diviner/workflow/DivinerGenerateWorkflow.php

@@ -131,7 +131,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
     $this->buildAtomCache();
     $this->buildGraphCache();
 
-    $this->publishDocumentation();
+    $this->publishDocumentation($args->getArg('clean'));
   }
 
 /* -(  Atom Cache  )--------------------------------------------------------- */
@@ -439,13 +439,14 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
   }
 
 
-  private function publishDocumentation() {
+  private function publishDocumentation($clean) {
     $atom_cache = $this->getAtomCache();
     $graph_map = $atom_cache->getGraphMap();
 
     $this->log(pht('PUBLISHING DOCUMENTATION'));
 
     $publisher = new DivinerLivePublisher();
+    $publisher->setDropCaches($clean);
     $publisher->setConfig($this->getAllConfig());
     $publisher->setAtomCache($atom_cache);
     $publisher->setRenderer(new DivinerDefaultRenderer());

src/docs/book/user.book

@@ -1,4 +1,34 @@
 {
   "name" : "phabricator",
-  "root" : "../../../"
+  "title" : "Phabricator User Documentation",
+  "root" : "../../../",
+  "groups" : {
+    "intro" : {
+      "name" : "Introduction"
+    },
+    "config" : {
+      "name" : "Configuration"
+    },
+    "userguide" : {
+      "name" : "Application User Guides"
+    },
+    "differential" : {
+      "name" : "Differential (Code Review)"
+    },
+    "diffusion" : {
+      "name" : "Diffusion (Repository Browser)"
+    },
+    "maniphest" : {
+      "name" : "Maniphest (Task Tracking)"
+    },
+    "slowvote" : {
+      "name" : "Slowvote (Polls)"
+    },
+    "herald" : {
+      "name" : "Herald (Notifications)"
+    },
+    "phriction" : {
+      "name" : "Phriction (Wiki)"
+    }
+  }
 }

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -1342,6 +1342,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
         'type' => 'sql',
         'name' => $this->getPatchPath('20130531.filekeys.sql'),
       ),
+      '20130602.morediviner.sql' => array(
+        'type' => 'sql',
+        'name' => $this->getPatchPath('20130602.morediviner.sql'),
+      ),
     );
   }
 }