Add "live" publisher and storage to Diviner

Summary:
Ref T988. This adds basics for the non-static publishing target:

  - Storage (called "Live", e.g. `DivinerLiveAtom` to distinguish it from shared classes like `DivinerAtom`).
  - Mostly populate the storage.
  - Some minor fixes and improvements.

Test Plan: Generated docs, looked at DB, saw mostly-sensible output.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D5973

resources/sql/patches/20130519.diviner.sql

@@ -0,0 +1,38 @@
+CREATE TABLE {$NAMESPACE}_diviner.diviner_livebook (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  phid VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  name VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  viewPolicy VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  UNIQUE KEY (name),
+  UNIQUE KEY (phid)
+) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
+
+
+CREATE TABLE {$NAMESPACE}_diviner.diviner_livesymbol (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  phid VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  bookPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  context VARCHAR(255) COLLATE utf8_bin,
+  type VARCHAR (32) NOT NULL COLLATE utf8_bin,
+  name VARCHAR (255) NOT NULL COLLATE utf8_bin,
+  atomIndex INT UNSIGNED NOT NULL,
+  identityHash VARCHAR(12) NOT NULL COLLATE utf8_bin,
+  graphHash VARCHAR(33) COLLATE utf8_bin,
+  KEY (bookPHID, type, name(64), context(64), atomIndex),
+  KEY (name),
+  UNIQUE KEY (graphHash),
+  UNIQUE KEY (identityHash),
+  UNIQUE KEY (phid)
+) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
+
+CREATE TABLE {$NAMESPACE}_diviner.diviner_liveatom (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  symbolPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  content LONGTEXT NOT NULL COLLATE utf8_bin,
+  atomData LONGTEXT NOT NULL COLLATE utf8_bin,
+  UNIQUE KEY (symbolPHID)
+) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
+
+

src/__phutil_library_map__.php

@@ -505,11 +505,16 @@ phutil_register_library_map(array(
     'DivinerAtomRef' => 'applications/diviner/atom/DivinerAtomRef.php',
     'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php',
     'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php',
+    'DivinerDAO' => 'applications/diviner/storage/DivinerDAO.php',
     'DivinerDefaultRenderer' => 'applications/diviner/renderer/DivinerDefaultRenderer.php',
     'DivinerDiskCache' => 'applications/diviner/cache/DivinerDiskCache.php',
     'DivinerFileAtomizer' => 'applications/diviner/atomizer/DivinerFileAtomizer.php',
     'DivinerGenerateWorkflow' => 'applications/diviner/workflow/DivinerGenerateWorkflow.php',
     'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
+    'DivinerLiveAtom' => 'applications/diviner/storage/DivinerLiveAtom.php',
+    'DivinerLiveBook' => 'applications/diviner/storage/DivinerLiveBook.php',
+    'DivinerLivePublisher' => 'applications/diviner/publisher/DivinerLivePublisher.php',
+    'DivinerLiveSymbol' => 'applications/diviner/storage/DivinerLiveSymbol.php',
     'DivinerPublishCache' => 'applications/diviner/cache/DivinerPublishCache.php',
     'DivinerPublisher' => 'applications/diviner/publisher/DivinerPublisher.php',
     'DivinerRemarkupRuleSymbol' => 'applications/diviner/markup/DivinerRemarkupRuleSymbol.php',
@@ -2289,10 +2294,23 @@ phutil_register_library_map(array(
     'DivinerArticleAtomizer' => 'DivinerAtomizer',
     'DivinerAtomCache' => 'DivinerDiskCache',
     'DivinerAtomizeWorkflow' => 'DivinerWorkflow',
+    'DivinerDAO' => 'PhabricatorLiskDAO',
     'DivinerDefaultRenderer' => 'DivinerRenderer',
     'DivinerFileAtomizer' => 'DivinerAtomizer',
     'DivinerGenerateWorkflow' => 'DivinerWorkflow',
     'DivinerListController' => 'PhabricatorController',
+    'DivinerLiveAtom' =>
+    array(
+      0 => 'DivinerDAO',
+      1 => 'PhabricatorPolicyInterface',
+    ),
+    'DivinerLiveBook' =>
+    array(
+      0 => 'DivinerDAO',
+      1 => 'PhabricatorPolicyInterface',
+    ),
+    'DivinerLivePublisher' => 'DivinerPublisher',
+    'DivinerLiveSymbol' => 'DivinerDAO',
     'DivinerPublishCache' => 'DivinerDiskCache',
     'DivinerRemarkupRuleSymbol' => 'PhutilRemarkupRule',
     'DivinerStaticPublisher' => 'DivinerPublisher',

src/applications/diviner/publisher/DivinerLivePublisher.php

@@ -0,0 +1,121 @@
+<?php
+
+final class DivinerLivePublisher extends DivinerPublisher {
+
+  private $book;
+
+  private function loadBook() {
+    if (!$this->book) {
+      $book_name = $this->getConfig('name');
+
+      $book = id(new DivinerLiveBook())->loadOneWhere(
+        'name = %s',
+        $book_name);
+      if (!$book) {
+        $book = id(new DivinerLiveBook())
+          ->setName($book_name)
+          ->setViewPolicy(PhabricatorPolicies::POLICY_USER)
+          ->save();
+      }
+
+      $this->book = $book;
+    }
+    return $this->book;
+  }
+
+  private function loadSymbolForAtom(DivinerAtom $atom) {
+    $symbol = id(new DivinerLiveSymbol())->loadOneWhere(
+      'bookPHID = %s AND type = %s AND name = %s AND context = %ns
+        AND atomIndex = %d',
+      $this->loadBook()->getPHID(),
+      $atom->getType(),
+      $atom->getName(),
+      $atom->getContext(),
+      $this->getAtomSimilarIndex($atom));
+
+    if ($symbol) {
+      return $symbol;
+    }
+
+    return id(new DivinerLiveSymbol())
+      ->setBookPHID($this->loadBook()->getPHID())
+      ->setType($atom->getType())
+      ->setName($atom->getName())
+      ->setContext($atom->getContext())
+      ->setAtomIndex($this->getAtomSimilarIndex($atom));
+  }
+
+  private function loadAtomStorageForSymbol(DivinerLiveSymbol $symbol) {
+    $storage = id(new DivinerLiveAtom())->loadOneWhere(
+      'symbolPHID = %s',
+      $symbol->getPHID());
+
+    if ($storage) {
+      return $storage;
+    }
+
+    return id(new DivinerLiveAtom())
+      ->setSymbolPHID($symbol->getPHID());
+  }
+
+  protected function loadAllPublishedHashes() {
+    $symbols = id(new DivinerLiveSymbol())->loadAllWhere(
+      'bookPHID = %s AND graphHash IS NOT NULL',
+      $this->loadBook()->getPHID());
+
+    return mpull($symbols, 'getGraphHash');
+  }
+
+  protected function deleteDocumentsByHash(array $hashes) {
+    $atom_table = new DivinerLiveAtom();
+    $symbol_table = new DivinerLiveSymbol();
+
+    $conn_w = $symbol_table->establishConnection('w');
+
+    $strings = array();
+    foreach ($hashes as $hash) {
+      $strings[] = qsprintf($conn_w, '%s', $hash);
+    }
+
+    foreach (PhabricatorLiskDAO::chunkSQL($strings, ', ') as $chunk) {
+      queryfx(
+        $conn_w,
+        'UPDATE %T SET graphHash = NULL WHERE graphHash IN (%Q)',
+        $symbol_table->getTableName(),
+        $chunk);
+    }
+
+    queryfx(
+      $conn_w,
+      'DELETE a FROM %T a LEFT JOIN %T s
+        ON a.symbolPHID = s.phid
+        WHERE s.graphHash IS NULL',
+      $atom_table->getTableName(),
+      $symbol_table->getTableName());
+  }
+
+  protected function createDocumentsByHash(array $hashes) {
+    foreach ($hashes as $hash) {
+      $atom = $this->getAtomFromGraphHash($hash);
+
+      $symbol = $this->loadSymbolForAtom($atom);
+      $symbol->setGraphHash($hash)->save();
+
+      if ($this->shouldGenerateDocumentForAtom($atom)) {
+        $content = $this->getRenderer()->renderAtom($atom);
+
+        $this->loadAtomStorageForSymbol($symbol)
+          ->setAtomData($atom->toDictionary())
+          ->setContent(phutil_safe_html($content))
+          ->save();
+      }
+    }
+  }
+
+  public function findAtomByRef(DivinerAtomRef $ref) {
+    // TODO: Actually implement this.
+
+    return null;
+  }
+
+}

src/applications/diviner/publisher/DivinerPublisher.php

@@ -86,7 +86,7 @@ abstract class DivinerPublisher {
   protected function getAtomSimilarIndex(DivinerAtom $atom) {
     $atoms = $this->getSimilarAtoms($atom);
     if (count($atoms) == 1) {
-      return null;
+      return 0;
     }
 
     $index = 1;

src/applications/diviner/publisher/DivinerStaticPublisher.php

@@ -53,7 +53,6 @@ final class DivinerStaticPublisher extends DivinerPublisher {
       }
 
       $cache->removeAtomPathsFromCache($hash);
-      $cache->deleteRenderCache($hash);
       $cache->deleteAtomFromIndex($hash);
     }
   }

src/applications/diviner/renderer/DivinerDefaultRenderer.php

@@ -155,7 +155,7 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
             'class' => 'atom-index-item',
           ),
           array(
-            $ref->getName(),
+            $this->renderAtomRefLink($ref),
             ' - ',
             $ref->getSummary(),
           ));
@@ -227,8 +227,13 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
   }
 
   public function getHrefForAtomRef(DivinerAtomRef $ref) {
+    $depth = 1;
+
     $atom = $this->peekAtomStack();
-    $depth = $this->getAtomHrefDepth($atom);
+    if ($atom) {
+      $depth = $this->getAtomHrefDepth($atom);
+    }
+
     $href = str_repeat('../', $depth);
 
     $book = $ref->getBook();
@@ -240,9 +245,19 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
     if ($context !== null) {
       $href .= $context.'/';
     }
-    $href .= $name.'/';
+    $href .= $name.'/index.html';
 
     return $href;
   }
 
+  protected function renderAtomRefLink(DivinerAtomRef $ref) {
+    return phutil_tag(
+      'a',
+      array(
+        'href' => $this->getHrefForAtomRef($ref),
+      ),
+      $ref->getTitle());
+  }
+
+
 }

src/applications/diviner/renderer/DivinerRenderer.php

@@ -3,7 +3,7 @@
 abstract class DivinerRenderer {
 
   private $publisher;
-  private $atomStack;
+  private $atomStack = array();
 
   public function setPublisher($publisher) {
     $this->publisher = $publisher;

src/applications/diviner/storage/DivinerDAO.php

@@ -0,0 +1,9 @@
+<?php
+
+abstract class DivinerDAO extends PhabricatorLiskDAO {
+
+  public function getApplicationName() {
+    return 'diviner';
+  }
+
+}

src/applications/diviner/storage/DivinerLiveAtom.php

@@ -0,0 +1,18 @@
+<?php
+
+final class DivinerLiveAtom extends DivinerDAO {
+
+  protected $symbolPHID;
+  protected $content;
+  protected $atomData;
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_SERIALIZATION => array(
+        'atomData' => self::SERIALIZATION_JSON,
+      ),
+    ) + parent::getConfiguration();
+  }
+
+}

src/applications/diviner/storage/DivinerLiveBook.php

@@ -0,0 +1,37 @@
+<?php
+
+final class DivinerLiveBook extends DivinerDAO
+  implements PhabricatorPolicyInterface {
+
+  protected $phid;
+  protected $name;
+  protected $viewPolicy;
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_AUX_PHID => true,
+    ) + parent::getConfiguration();
+  }
+
+  public function generatePHID() {
+    return PhabricatorPHID::generateNewPHID(
+      PhabricatorPHIDConstants::PHID_TYPE_BOOK);
+  }
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    return $this->viewPolicy;
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return false;
+  }
+
+}

src/applications/diviner/storage/DivinerLiveSymbol.php

@@ -0,0 +1,48 @@
+<?php
+
+final class DivinerLiveSymbol extends DivinerDAO {
+
+  protected $phid;
+  protected $bookPHID;
+  protected $context;
+  protected $type;
+  protected $name;
+  protected $atomIndex;
+  protected $graphHash;
+  protected $identityHash;
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_AUX_PHID => true,
+      self::CONFIG_TIMESTAMPS => false,
+    ) + parent::getConfiguration();
+  }
+
+  public function generatePHID() {
+    return PhabricatorPHID::generateNewPHID(
+      PhabricatorPHIDConstants::PHID_TYPE_ATOM);
+  }
+
+  public function save() {
+
+    // NOTE: The identity hash is just a sanity check because the unique tuple
+    // on this table is way way too long to fit into a normal UNIQUE KEY. We
+    // don't use it directly, but its existence prevents duplicate records.
+
+    if (!$this->identityHash) {
+      $this->identityHash = PhabricatorHash::digestForIndex(
+        serialize(
+          array(
+            'bookPHID' => $this->getBookPHID(),
+            'context'  => $this->getContext(),
+            'type'     => $this->getType(),
+            'name'     => $this->getName(),
+            'index'    => $this->getAtomIndex(),
+          )));
+    }
+
+    return parent::save();
+  }
+
+
+}

src/applications/diviner/workflow/DivinerGenerateWorkflow.php

@@ -445,7 +445,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
 
     $this->log(pht('PUBLISHING DOCUMENTATION'));
 
-    $publisher = new DivinerStaticPublisher();
+    $publisher = new DivinerLivePublisher();
     $publisher->setConfig($this->getAllConfig());
     $publisher->setAtomCache($atom_cache);
     $publisher->setRenderer(new DivinerDefaultRenderer());

src/applications/phid/PhabricatorPHIDConstants.php

@@ -44,6 +44,9 @@ final class PhabricatorPHIDConstants {
   const PHID_TYPE_XCMT    = 'XCMT';
   const PHID_TYPE_XUSR    = 'XUSR';
 
+  const PHID_TYPE_BOOK    = 'BOOK';
+  const PHID_TYPE_ATOM    = 'ATOM';
+
   const PHID_TYPE_VOID    = 'VOID';
   const PHID_VOID         = 'PHID-VOID-00000000000000000000';
 

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -187,6 +187,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
         'type' => 'db',
         'name' => 'phrequent',
       ),
+      'db.diviner' => array(
+        'type' => 'db',
+        'name' => 'diviner',
+      ),
       '0000.legacy.sql' => array(
         'type'    => 'sql',
         'name'    => $this->getPatchPath('0000.legacy.sql'),
@@ -1298,6 +1302,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
         'type' => 'sql',
         'name' => $this->getPatchPath('20130513.receviedmailstatus.sql'),
       ),
+      '20130519.diviner.sql' => array(
+        'type' => 'sql',
+        'name' => $this->getPatchPath('20130519.diviner.sql'),
+      ),
     );
   }
 }