mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-10 00:42:41 +01:00
Add "live" publisher and storage to Diviner
Summary: Ref T988. This adds basics for the non-static publishing target: - Storage (called "Live", e.g. `DivinerLiveAtom` to distinguish it from shared classes like `DivinerAtom`). - Mostly populate the storage. - Some minor fixes and improvements. Test Plan: Generated docs, looked at DB, saw mostly-sensible output. Reviewers: btrahan Reviewed By: btrahan CC: aran Maniphest Tasks: T988 Differential Revision: https://secure.phabricator.com/D5973
This commit is contained in:
parent
e01ceaa07f
commit
b0a5f42244
14 changed files with 321 additions and 7 deletions
38
resources/sql/patches/20130519.diviner.sql
Normal file
38
resources/sql/patches/20130519.diviner.sql
Normal file
|
@ -0,0 +1,38 @@
|
|||
CREATE TABLE {$NAMESPACE}_diviner.diviner_livebook (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
phid VARCHAR(64) NOT NULL COLLATE utf8_bin,
|
||||
name VARCHAR(64) NOT NULL COLLATE utf8_bin,
|
||||
viewPolicy VARCHAR(64) NOT NULL COLLATE utf8_bin,
|
||||
dateCreated INT UNSIGNED NOT NULL,
|
||||
dateModified INT UNSIGNED NOT NULL,
|
||||
UNIQUE KEY (name),
|
||||
UNIQUE KEY (phid)
|
||||
) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
|
||||
|
||||
|
||||
CREATE TABLE {$NAMESPACE}_diviner.diviner_livesymbol (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
phid VARCHAR(64) NOT NULL COLLATE utf8_bin,
|
||||
bookPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
|
||||
context VARCHAR(255) COLLATE utf8_bin,
|
||||
type VARCHAR (32) NOT NULL COLLATE utf8_bin,
|
||||
name VARCHAR (255) NOT NULL COLLATE utf8_bin,
|
||||
atomIndex INT UNSIGNED NOT NULL,
|
||||
identityHash VARCHAR(12) NOT NULL COLLATE utf8_bin,
|
||||
graphHash VARCHAR(33) COLLATE utf8_bin,
|
||||
KEY (bookPHID, type, name(64), context(64), atomIndex),
|
||||
KEY (name),
|
||||
UNIQUE KEY (graphHash),
|
||||
UNIQUE KEY (identityHash),
|
||||
UNIQUE KEY (phid)
|
||||
) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
|
||||
|
||||
CREATE TABLE {$NAMESPACE}_diviner.diviner_liveatom (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
symbolPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
|
||||
content LONGTEXT NOT NULL COLLATE utf8_bin,
|
||||
atomData LONGTEXT NOT NULL COLLATE utf8_bin,
|
||||
UNIQUE KEY (symbolPHID)
|
||||
) ENGINE=InnoDB, DEFAULT CHARSET = utf8;
|
||||
|
||||
|
|
@ -505,11 +505,16 @@ phutil_register_library_map(array(
|
|||
'DivinerAtomRef' => 'applications/diviner/atom/DivinerAtomRef.php',
|
||||
'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php',
|
||||
'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php',
|
||||
'DivinerDAO' => 'applications/diviner/storage/DivinerDAO.php',
|
||||
'DivinerDefaultRenderer' => 'applications/diviner/renderer/DivinerDefaultRenderer.php',
|
||||
'DivinerDiskCache' => 'applications/diviner/cache/DivinerDiskCache.php',
|
||||
'DivinerFileAtomizer' => 'applications/diviner/atomizer/DivinerFileAtomizer.php',
|
||||
'DivinerGenerateWorkflow' => 'applications/diviner/workflow/DivinerGenerateWorkflow.php',
|
||||
'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
|
||||
'DivinerLiveAtom' => 'applications/diviner/storage/DivinerLiveAtom.php',
|
||||
'DivinerLiveBook' => 'applications/diviner/storage/DivinerLiveBook.php',
|
||||
'DivinerLivePublisher' => 'applications/diviner/publisher/DivinerLivePublisher.php',
|
||||
'DivinerLiveSymbol' => 'applications/diviner/storage/DivinerLiveSymbol.php',
|
||||
'DivinerPublishCache' => 'applications/diviner/cache/DivinerPublishCache.php',
|
||||
'DivinerPublisher' => 'applications/diviner/publisher/DivinerPublisher.php',
|
||||
'DivinerRemarkupRuleSymbol' => 'applications/diviner/markup/DivinerRemarkupRuleSymbol.php',
|
||||
|
@ -2289,10 +2294,23 @@ phutil_register_library_map(array(
|
|||
'DivinerArticleAtomizer' => 'DivinerAtomizer',
|
||||
'DivinerAtomCache' => 'DivinerDiskCache',
|
||||
'DivinerAtomizeWorkflow' => 'DivinerWorkflow',
|
||||
'DivinerDAO' => 'PhabricatorLiskDAO',
|
||||
'DivinerDefaultRenderer' => 'DivinerRenderer',
|
||||
'DivinerFileAtomizer' => 'DivinerAtomizer',
|
||||
'DivinerGenerateWorkflow' => 'DivinerWorkflow',
|
||||
'DivinerListController' => 'PhabricatorController',
|
||||
'DivinerLiveAtom' =>
|
||||
array(
|
||||
0 => 'DivinerDAO',
|
||||
1 => 'PhabricatorPolicyInterface',
|
||||
),
|
||||
'DivinerLiveBook' =>
|
||||
array(
|
||||
0 => 'DivinerDAO',
|
||||
1 => 'PhabricatorPolicyInterface',
|
||||
),
|
||||
'DivinerLivePublisher' => 'DivinerPublisher',
|
||||
'DivinerLiveSymbol' => 'DivinerDAO',
|
||||
'DivinerPublishCache' => 'DivinerDiskCache',
|
||||
'DivinerRemarkupRuleSymbol' => 'PhutilRemarkupRule',
|
||||
'DivinerStaticPublisher' => 'DivinerPublisher',
|
||||
|
|
121
src/applications/diviner/publisher/DivinerLivePublisher.php
Normal file
121
src/applications/diviner/publisher/DivinerLivePublisher.php
Normal file
|
@ -0,0 +1,121 @@
|
|||
<?php
|
||||
|
||||
final class DivinerLivePublisher extends DivinerPublisher {
|
||||
|
||||
private $book;
|
||||
|
||||
private function loadBook() {
|
||||
if (!$this->book) {
|
||||
$book_name = $this->getConfig('name');
|
||||
|
||||
$book = id(new DivinerLiveBook())->loadOneWhere(
|
||||
'name = %s',
|
||||
$book_name);
|
||||
if (!$book) {
|
||||
$book = id(new DivinerLiveBook())
|
||||
->setName($book_name)
|
||||
->setViewPolicy(PhabricatorPolicies::POLICY_USER)
|
||||
->save();
|
||||
}
|
||||
|
||||
$this->book = $book;
|
||||
}
|
||||
return $this->book;
|
||||
}
|
||||
|
||||
private function loadSymbolForAtom(DivinerAtom $atom) {
|
||||
$symbol = id(new DivinerLiveSymbol())->loadOneWhere(
|
||||
'bookPHID = %s AND type = %s AND name = %s AND context = %ns
|
||||
AND atomIndex = %d',
|
||||
$this->loadBook()->getPHID(),
|
||||
$atom->getType(),
|
||||
$atom->getName(),
|
||||
$atom->getContext(),
|
||||
$this->getAtomSimilarIndex($atom));
|
||||
|
||||
if ($symbol) {
|
||||
return $symbol;
|
||||
}
|
||||
|
||||
return id(new DivinerLiveSymbol())
|
||||
->setBookPHID($this->loadBook()->getPHID())
|
||||
->setType($atom->getType())
|
||||
->setName($atom->getName())
|
||||
->setContext($atom->getContext())
|
||||
->setAtomIndex($this->getAtomSimilarIndex($atom));
|
||||
}
|
||||
|
||||
private function loadAtomStorageForSymbol(DivinerLiveSymbol $symbol) {
|
||||
$storage = id(new DivinerLiveAtom())->loadOneWhere(
|
||||
'symbolPHID = %s',
|
||||
$symbol->getPHID());
|
||||
|
||||
if ($storage) {
|
||||
return $storage;
|
||||
}
|
||||
|
||||
return id(new DivinerLiveAtom())
|
||||
->setSymbolPHID($symbol->getPHID());
|
||||
}
|
||||
|
||||
protected function loadAllPublishedHashes() {
|
||||
$symbols = id(new DivinerLiveSymbol())->loadAllWhere(
|
||||
'bookPHID = %s AND graphHash IS NOT NULL',
|
||||
$this->loadBook()->getPHID());
|
||||
|
||||
return mpull($symbols, 'getGraphHash');
|
||||
}
|
||||
|
||||
protected function deleteDocumentsByHash(array $hashes) {
|
||||
$atom_table = new DivinerLiveAtom();
|
||||
$symbol_table = new DivinerLiveSymbol();
|
||||
|
||||
$conn_w = $symbol_table->establishConnection('w');
|
||||
|
||||
$strings = array();
|
||||
foreach ($hashes as $hash) {
|
||||
$strings[] = qsprintf($conn_w, '%s', $hash);
|
||||
}
|
||||
|
||||
foreach (PhabricatorLiskDAO::chunkSQL($strings, ', ') as $chunk) {
|
||||
queryfx(
|
||||
$conn_w,
|
||||
'UPDATE %T SET graphHash = NULL WHERE graphHash IN (%Q)',
|
||||
$symbol_table->getTableName(),
|
||||
$chunk);
|
||||
}
|
||||
|
||||
queryfx(
|
||||
$conn_w,
|
||||
'DELETE a FROM %T a LEFT JOIN %T s
|
||||
ON a.symbolPHID = s.phid
|
||||
WHERE s.graphHash IS NULL',
|
||||
$atom_table->getTableName(),
|
||||
$symbol_table->getTableName());
|
||||
}
|
||||
|
||||
protected function createDocumentsByHash(array $hashes) {
|
||||
foreach ($hashes as $hash) {
|
||||
$atom = $this->getAtomFromGraphHash($hash);
|
||||
|
||||
$symbol = $this->loadSymbolForAtom($atom);
|
||||
$symbol->setGraphHash($hash)->save();
|
||||
|
||||
if ($this->shouldGenerateDocumentForAtom($atom)) {
|
||||
$content = $this->getRenderer()->renderAtom($atom);
|
||||
|
||||
$this->loadAtomStorageForSymbol($symbol)
|
||||
->setAtomData($atom->toDictionary())
|
||||
->setContent(phutil_safe_html($content))
|
||||
->save();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function findAtomByRef(DivinerAtomRef $ref) {
|
||||
// TODO: Actually implement this.
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
|
@ -86,7 +86,7 @@ abstract class DivinerPublisher {
|
|||
protected function getAtomSimilarIndex(DivinerAtom $atom) {
|
||||
$atoms = $this->getSimilarAtoms($atom);
|
||||
if (count($atoms) == 1) {
|
||||
return null;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$index = 1;
|
||||
|
|
|
@ -53,7 +53,6 @@ final class DivinerStaticPublisher extends DivinerPublisher {
|
|||
}
|
||||
|
||||
$cache->removeAtomPathsFromCache($hash);
|
||||
$cache->deleteRenderCache($hash);
|
||||
$cache->deleteAtomFromIndex($hash);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -155,7 +155,7 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
|
|||
'class' => 'atom-index-item',
|
||||
),
|
||||
array(
|
||||
$ref->getName(),
|
||||
$this->renderAtomRefLink($ref),
|
||||
' - ',
|
||||
$ref->getSummary(),
|
||||
));
|
||||
|
@ -227,8 +227,13 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
|
|||
}
|
||||
|
||||
public function getHrefForAtomRef(DivinerAtomRef $ref) {
|
||||
$depth = 1;
|
||||
|
||||
$atom = $this->peekAtomStack();
|
||||
$depth = $this->getAtomHrefDepth($atom);
|
||||
if ($atom) {
|
||||
$depth = $this->getAtomHrefDepth($atom);
|
||||
}
|
||||
|
||||
$href = str_repeat('../', $depth);
|
||||
|
||||
$book = $ref->getBook();
|
||||
|
@ -240,9 +245,19 @@ final class DivinerDefaultRenderer extends DivinerRenderer {
|
|||
if ($context !== null) {
|
||||
$href .= $context.'/';
|
||||
}
|
||||
$href .= $name.'/';
|
||||
$href .= $name.'/index.html';
|
||||
|
||||
return $href;
|
||||
}
|
||||
|
||||
protected function renderAtomRefLink(DivinerAtomRef $ref) {
|
||||
return phutil_tag(
|
||||
'a',
|
||||
array(
|
||||
'href' => $this->getHrefForAtomRef($ref),
|
||||
),
|
||||
$ref->getTitle());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
abstract class DivinerRenderer {
|
||||
|
||||
private $publisher;
|
||||
private $atomStack;
|
||||
private $atomStack = array();
|
||||
|
||||
public function setPublisher($publisher) {
|
||||
$this->publisher = $publisher;
|
||||
|
|
9
src/applications/diviner/storage/DivinerDAO.php
Normal file
9
src/applications/diviner/storage/DivinerDAO.php
Normal file
|
@ -0,0 +1,9 @@
|
|||
<?php
|
||||
|
||||
abstract class DivinerDAO extends PhabricatorLiskDAO {
|
||||
|
||||
public function getApplicationName() {
|
||||
return 'diviner';
|
||||
}
|
||||
|
||||
}
|
18
src/applications/diviner/storage/DivinerLiveAtom.php
Normal file
18
src/applications/diviner/storage/DivinerLiveAtom.php
Normal file
|
@ -0,0 +1,18 @@
|
|||
<?php
|
||||
|
||||
final class DivinerLiveAtom extends DivinerDAO {
|
||||
|
||||
protected $symbolPHID;
|
||||
protected $content;
|
||||
protected $atomData;
|
||||
|
||||
public function getConfiguration() {
|
||||
return array(
|
||||
self::CONFIG_TIMESTAMPS => false,
|
||||
self::CONFIG_SERIALIZATION => array(
|
||||
'atomData' => self::SERIALIZATION_JSON,
|
||||
),
|
||||
) + parent::getConfiguration();
|
||||
}
|
||||
|
||||
}
|
37
src/applications/diviner/storage/DivinerLiveBook.php
Normal file
37
src/applications/diviner/storage/DivinerLiveBook.php
Normal file
|
@ -0,0 +1,37 @@
|
|||
<?php
|
||||
|
||||
final class DivinerLiveBook extends DivinerDAO
|
||||
implements PhabricatorPolicyInterface {
|
||||
|
||||
protected $phid;
|
||||
protected $name;
|
||||
protected $viewPolicy;
|
||||
|
||||
public function getConfiguration() {
|
||||
return array(
|
||||
self::CONFIG_AUX_PHID => true,
|
||||
) + parent::getConfiguration();
|
||||
}
|
||||
|
||||
public function generatePHID() {
|
||||
return PhabricatorPHID::generateNewPHID(
|
||||
PhabricatorPHIDConstants::PHID_TYPE_BOOK);
|
||||
}
|
||||
|
||||
/* -( PhabricatorPolicyInterface )----------------------------------------- */
|
||||
|
||||
public function getCapabilities() {
|
||||
return array(
|
||||
PhabricatorPolicyCapability::CAN_VIEW,
|
||||
);
|
||||
}
|
||||
|
||||
public function getPolicy($capability) {
|
||||
return $this->viewPolicy;
|
||||
}
|
||||
|
||||
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
48
src/applications/diviner/storage/DivinerLiveSymbol.php
Normal file
48
src/applications/diviner/storage/DivinerLiveSymbol.php
Normal file
|
@ -0,0 +1,48 @@
|
|||
<?php
|
||||
|
||||
final class DivinerLiveSymbol extends DivinerDAO {
|
||||
|
||||
protected $phid;
|
||||
protected $bookPHID;
|
||||
protected $context;
|
||||
protected $type;
|
||||
protected $name;
|
||||
protected $atomIndex;
|
||||
protected $graphHash;
|
||||
protected $identityHash;
|
||||
|
||||
public function getConfiguration() {
|
||||
return array(
|
||||
self::CONFIG_AUX_PHID => true,
|
||||
self::CONFIG_TIMESTAMPS => false,
|
||||
) + parent::getConfiguration();
|
||||
}
|
||||
|
||||
public function generatePHID() {
|
||||
return PhabricatorPHID::generateNewPHID(
|
||||
PhabricatorPHIDConstants::PHID_TYPE_ATOM);
|
||||
}
|
||||
|
||||
public function save() {
|
||||
|
||||
// NOTE: The identity hash is just a sanity check because the unique tuple
|
||||
// on this table is way way too long to fit into a normal UNIQUE KEY. We
|
||||
// don't use it directly, but its existence prevents duplicate records.
|
||||
|
||||
if (!$this->identityHash) {
|
||||
$this->identityHash = PhabricatorHash::digestForIndex(
|
||||
serialize(
|
||||
array(
|
||||
'bookPHID' => $this->getBookPHID(),
|
||||
'context' => $this->getContext(),
|
||||
'type' => $this->getType(),
|
||||
'name' => $this->getName(),
|
||||
'index' => $this->getAtomIndex(),
|
||||
)));
|
||||
}
|
||||
|
||||
return parent::save();
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -445,7 +445,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
|
|||
|
||||
$this->log(pht('PUBLISHING DOCUMENTATION'));
|
||||
|
||||
$publisher = new DivinerStaticPublisher();
|
||||
$publisher = new DivinerLivePublisher();
|
||||
$publisher->setConfig($this->getAllConfig());
|
||||
$publisher->setAtomCache($atom_cache);
|
||||
$publisher->setRenderer(new DivinerDefaultRenderer());
|
||||
|
|
|
@ -44,6 +44,9 @@ final class PhabricatorPHIDConstants {
|
|||
const PHID_TYPE_XCMT = 'XCMT';
|
||||
const PHID_TYPE_XUSR = 'XUSR';
|
||||
|
||||
const PHID_TYPE_BOOK = 'BOOK';
|
||||
const PHID_TYPE_ATOM = 'ATOM';
|
||||
|
||||
const PHID_TYPE_VOID = 'VOID';
|
||||
const PHID_VOID = 'PHID-VOID-00000000000000000000';
|
||||
|
||||
|
|
|
@ -187,6 +187,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
|
|||
'type' => 'db',
|
||||
'name' => 'phrequent',
|
||||
),
|
||||
'db.diviner' => array(
|
||||
'type' => 'db',
|
||||
'name' => 'diviner',
|
||||
),
|
||||
'0000.legacy.sql' => array(
|
||||
'type' => 'sql',
|
||||
'name' => $this->getPatchPath('0000.legacy.sql'),
|
||||
|
@ -1298,6 +1302,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
|
|||
'type' => 'sql',
|
||||
'name' => $this->getPatchPath('20130513.receviedmailstatus.sql'),
|
||||
),
|
||||
'20130519.diviner.sql' => array(
|
||||
'type' => 'sql',
|
||||
'name' => $this->getPatchPath('20130519.diviner.sql'),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue