1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00

Add "live" publisher and storage to Diviner

Summary:
Ref T988. This adds basics for the non-static publishing target:

  - Storage (called "Live", e.g. `DivinerLiveAtom` to distinguish it from shared classes like `DivinerAtom`).
  - Mostly populate the storage.
  - Some minor fixes and improvements.

Test Plan: Generated docs, looked at DB, saw mostly-sensible output.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D5973
This commit is contained in:
epriestley 2013-05-20 10:18:26 -07:00
parent e01ceaa07f
commit b0a5f42244
14 changed files with 321 additions and 7 deletions

View file

@ -0,0 +1,38 @@
CREATE TABLE {$NAMESPACE}_diviner.diviner_livebook (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
phid VARCHAR(64) NOT NULL COLLATE utf8_bin,
name VARCHAR(64) NOT NULL COLLATE utf8_bin,
viewPolicy VARCHAR(64) NOT NULL COLLATE utf8_bin,
dateCreated INT UNSIGNED NOT NULL,
dateModified INT UNSIGNED NOT NULL,
UNIQUE KEY (name),
UNIQUE KEY (phid)
) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
CREATE TABLE {$NAMESPACE}_diviner.diviner_livesymbol (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
phid VARCHAR(64) NOT NULL COLLATE utf8_bin,
bookPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
context VARCHAR(255) COLLATE utf8_bin,
type VARCHAR (32) NOT NULL COLLATE utf8_bin,
name VARCHAR (255) NOT NULL COLLATE utf8_bin,
atomIndex INT UNSIGNED NOT NULL,
identityHash VARCHAR(12) NOT NULL COLLATE utf8_bin,
graphHash VARCHAR(33) COLLATE utf8_bin,
KEY (bookPHID, type, name(64), context(64), atomIndex),
KEY (name),
UNIQUE KEY (graphHash),
UNIQUE KEY (identityHash),
UNIQUE KEY (phid)
) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
CREATE TABLE {$NAMESPACE}_diviner.diviner_liveatom (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
symbolPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
content LONGTEXT NOT NULL COLLATE utf8_bin,
atomData LONGTEXT NOT NULL COLLATE utf8_bin,
UNIQUE KEY (symbolPHID)
) ENGINE=InnoDB, DEFAULT CHARSET = utf8;

View file

@ -505,11 +505,16 @@ phutil_register_library_map(array(
'DivinerAtomRef' => 'applications/diviner/atom/DivinerAtomRef.php',
'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php',
'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php',
'DivinerDAO' => 'applications/diviner/storage/DivinerDAO.php',
'DivinerDefaultRenderer' => 'applications/diviner/renderer/DivinerDefaultRenderer.php',
'DivinerDiskCache' => 'applications/diviner/cache/DivinerDiskCache.php',
'DivinerFileAtomizer' => 'applications/diviner/atomizer/DivinerFileAtomizer.php',
'DivinerGenerateWorkflow' => 'applications/diviner/workflow/DivinerGenerateWorkflow.php',
'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
'DivinerLiveAtom' => 'applications/diviner/storage/DivinerLiveAtom.php',
'DivinerLiveBook' => 'applications/diviner/storage/DivinerLiveBook.php',
'DivinerLivePublisher' => 'applications/diviner/publisher/DivinerLivePublisher.php',
'DivinerLiveSymbol' => 'applications/diviner/storage/DivinerLiveSymbol.php',
'DivinerPublishCache' => 'applications/diviner/cache/DivinerPublishCache.php',
'DivinerPublisher' => 'applications/diviner/publisher/DivinerPublisher.php',
'DivinerRemarkupRuleSymbol' => 'applications/diviner/markup/DivinerRemarkupRuleSymbol.php',
@ -2289,10 +2294,23 @@ phutil_register_library_map(array(
'DivinerArticleAtomizer' => 'DivinerAtomizer',
'DivinerAtomCache' => 'DivinerDiskCache',
'DivinerAtomizeWorkflow' => 'DivinerWorkflow',
'DivinerDAO' => 'PhabricatorLiskDAO',
'DivinerDefaultRenderer' => 'DivinerRenderer',
'DivinerFileAtomizer' => 'DivinerAtomizer',
'DivinerGenerateWorkflow' => 'DivinerWorkflow',
'DivinerListController' => 'PhabricatorController',
'DivinerLiveAtom' =>
array(
0 => 'DivinerDAO',
1 => 'PhabricatorPolicyInterface',
),
'DivinerLiveBook' =>
array(
0 => 'DivinerDAO',
1 => 'PhabricatorPolicyInterface',
),
'DivinerLivePublisher' => 'DivinerPublisher',
'DivinerLiveSymbol' => 'DivinerDAO',
'DivinerPublishCache' => 'DivinerDiskCache',
'DivinerRemarkupRuleSymbol' => 'PhutilRemarkupRule',
'DivinerStaticPublisher' => 'DivinerPublisher',

View file

@ -0,0 +1,121 @@
<?php
final class DivinerLivePublisher extends DivinerPublisher {
private $book;
private function loadBook() {
if (!$this->book) {
$book_name = $this->getConfig('name');
$book = id(new DivinerLiveBook())->loadOneWhere(
'name = %s',
$book_name);
if (!$book) {
$book = id(new DivinerLiveBook())
->setName($book_name)
->setViewPolicy(PhabricatorPolicies::POLICY_USER)
->save();
}
$this->book = $book;
}
return $this->book;
}
private function loadSymbolForAtom(DivinerAtom $atom) {
$symbol = id(new DivinerLiveSymbol())->loadOneWhere(
'bookPHID = %s AND type = %s AND name = %s AND context = %ns
AND atomIndex = %d',
$this->loadBook()->getPHID(),
$atom->getType(),
$atom->getName(),
$atom->getContext(),
$this->getAtomSimilarIndex($atom));
if ($symbol) {
return $symbol;
}
return id(new DivinerLiveSymbol())
->setBookPHID($this->loadBook()->getPHID())
->setType($atom->getType())
->setName($atom->getName())
->setContext($atom->getContext())
->setAtomIndex($this->getAtomSimilarIndex($atom));
}
private function loadAtomStorageForSymbol(DivinerLiveSymbol $symbol) {
$storage = id(new DivinerLiveAtom())->loadOneWhere(
'symbolPHID = %s',
$symbol->getPHID());
if ($storage) {
return $storage;
}
return id(new DivinerLiveAtom())
->setSymbolPHID($symbol->getPHID());
}
protected function loadAllPublishedHashes() {
$symbols = id(new DivinerLiveSymbol())->loadAllWhere(
'bookPHID = %s AND graphHash IS NOT NULL',
$this->loadBook()->getPHID());
return mpull($symbols, 'getGraphHash');
}
protected function deleteDocumentsByHash(array $hashes) {
$atom_table = new DivinerLiveAtom();
$symbol_table = new DivinerLiveSymbol();
$conn_w = $symbol_table->establishConnection('w');
$strings = array();
foreach ($hashes as $hash) {
$strings[] = qsprintf($conn_w, '%s', $hash);
}
foreach (PhabricatorLiskDAO::chunkSQL($strings, ', ') as $chunk) {
queryfx(
$conn_w,
'UPDATE %T SET graphHash = NULL WHERE graphHash IN (%Q)',
$symbol_table->getTableName(),
$chunk);
}
queryfx(
$conn_w,
'DELETE a FROM %T a LEFT JOIN %T s
ON a.symbolPHID = s.phid
WHERE s.graphHash IS NULL',
$atom_table->getTableName(),
$symbol_table->getTableName());
}
protected function createDocumentsByHash(array $hashes) {
foreach ($hashes as $hash) {
$atom = $this->getAtomFromGraphHash($hash);
$symbol = $this->loadSymbolForAtom($atom);
$symbol->setGraphHash($hash)->save();
if ($this->shouldGenerateDocumentForAtom($atom)) {
$content = $this->getRenderer()->renderAtom($atom);
$this->loadAtomStorageForSymbol($symbol)
->setAtomData($atom->toDictionary())
->setContent(phutil_safe_html($content))
->save();
}
}
}
public function findAtomByRef(DivinerAtomRef $ref) {
// TODO: Actually implement this.
return null;
}
}

View file

@ -86,7 +86,7 @@ abstract class DivinerPublisher {
protected function getAtomSimilarIndex(DivinerAtom $atom) {
$atoms = $this->getSimilarAtoms($atom);
if (count($atoms) == 1) {
return null;
return 0;
}
$index = 1;

View file

@ -53,7 +53,6 @@ final class DivinerStaticPublisher extends DivinerPublisher {
}
$cache->removeAtomPathsFromCache($hash);
$cache->deleteRenderCache($hash);
$cache->deleteAtomFromIndex($hash);
}
}

View file

@ -155,7 +155,7 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
'class' => 'atom-index-item',
),
array(
$ref->getName(),
$this->renderAtomRefLink($ref),
' - ',
$ref->getSummary(),
));
@ -227,8 +227,13 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
}
public function getHrefForAtomRef(DivinerAtomRef $ref) {
$depth = 1;
$atom = $this->peekAtomStack();
$depth = $this->getAtomHrefDepth($atom);
if ($atom) {
$depth = $this->getAtomHrefDepth($atom);
}
$href = str_repeat('../', $depth);
$book = $ref->getBook();
@ -240,9 +245,19 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
if ($context !== null) {
$href .= $context.'/';
}
$href .= $name.'/';
$href .= $name.'/index.html';
return $href;
}
protected function renderAtomRefLink(DivinerAtomRef $ref) {
return phutil_tag(
'a',
array(
'href' => $this->getHrefForAtomRef($ref),
),
$ref->getTitle());
}
}

View file

@ -3,7 +3,7 @@
abstract class DivinerRenderer {
private $publisher;
private $atomStack;
private $atomStack = array();
public function setPublisher($publisher) {
$this->publisher = $publisher;

View file

@ -0,0 +1,9 @@
<?php
abstract class DivinerDAO extends PhabricatorLiskDAO {
public function getApplicationName() {
return 'diviner';
}
}

View file

@ -0,0 +1,18 @@
<?php
final class DivinerLiveAtom extends DivinerDAO {
protected $symbolPHID;
protected $content;
protected $atomData;
public function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_SERIALIZATION => array(
'atomData' => self::SERIALIZATION_JSON,
),
) + parent::getConfiguration();
}
}

View file

@ -0,0 +1,37 @@
<?php
final class DivinerLiveBook extends DivinerDAO
implements PhabricatorPolicyInterface {
protected $phid;
protected $name;
protected $viewPolicy;
public function getConfiguration() {
return array(
self::CONFIG_AUX_PHID => true,
) + parent::getConfiguration();
}
public function generatePHID() {
return PhabricatorPHID::generateNewPHID(
PhabricatorPHIDConstants::PHID_TYPE_BOOK);
}
/* -( PhabricatorPolicyInterface )----------------------------------------- */
public function getCapabilities() {
return array(
PhabricatorPolicyCapability::CAN_VIEW,
);
}
public function getPolicy($capability) {
return $this->viewPolicy;
}
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
return false;
}
}

View file

@ -0,0 +1,48 @@
<?php
final class DivinerLiveSymbol extends DivinerDAO {
protected $phid;
protected $bookPHID;
protected $context;
protected $type;
protected $name;
protected $atomIndex;
protected $graphHash;
protected $identityHash;
public function getConfiguration() {
return array(
self::CONFIG_AUX_PHID => true,
self::CONFIG_TIMESTAMPS => false,
) + parent::getConfiguration();
}
public function generatePHID() {
return PhabricatorPHID::generateNewPHID(
PhabricatorPHIDConstants::PHID_TYPE_ATOM);
}
public function save() {
// NOTE: The identity hash is just a sanity check because the unique tuple
// on this table is way way too long to fit into a normal UNIQUE KEY. We
// don't use it directly, but its existence prevents duplicate records.
if (!$this->identityHash) {
$this->identityHash = PhabricatorHash::digestForIndex(
serialize(
array(
'bookPHID' => $this->getBookPHID(),
'context' => $this->getContext(),
'type' => $this->getType(),
'name' => $this->getName(),
'index' => $this->getAtomIndex(),
)));
}
return parent::save();
}
}

View file

@ -445,7 +445,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
$this->log(pht('PUBLISHING DOCUMENTATION'));
$publisher = new DivinerStaticPublisher();
$publisher = new DivinerLivePublisher();
$publisher->setConfig($this->getAllConfig());
$publisher->setAtomCache($atom_cache);
$publisher->setRenderer(new DivinerDefaultRenderer());

View file

@ -44,6 +44,9 @@ final class PhabricatorPHIDConstants {
const PHID_TYPE_XCMT = 'XCMT';
const PHID_TYPE_XUSR = 'XUSR';
const PHID_TYPE_BOOK = 'BOOK';
const PHID_TYPE_ATOM = 'ATOM';
const PHID_TYPE_VOID = 'VOID';
const PHID_VOID = 'PHID-VOID-00000000000000000000';

View file

@ -187,6 +187,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
'type' => 'db',
'name' => 'phrequent',
),
'db.diviner' => array(
'type' => 'db',
'name' => 'diviner',
),
'0000.legacy.sql' => array(
'type' => 'sql',
'name' => $this->getPatchPath('0000.legacy.sql'),
@ -1298,6 +1302,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
'type' => 'sql',
'name' => $this->getPatchPath('20130513.receviedmailstatus.sql'),
),
'20130519.diviner.sql' => array(
'type' => 'sql',
'name' => $this->getPatchPath('20130519.diviner.sql'),
),
);
}
}