1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 14:52:41 +01:00

Provide an indirection layer between documents and the search engine

Summary:
In preparation for adding another search engine (see T355):

  - Rename "executor" to "engine".
  - Move all engine-specific operations into the engine. Specifically, this
means that indexing moves out of the document store and into the engine (it was
sort of silly where it was before).
  - Split choice of an engine into an overridable "selector" class, a base API,
and a concrete MySQL implementation (just like storage engine selection).
  - Make all callers go through the indirection layer.

The default selector just unconditionally selects the MySQL engine, but now
(with D786) I can build an Elastic Search engine and you guys can build a
multi-target engine if you want and I don't get there fast enough.

Test Plan:
  - Created a new document (task).
  - Searched for and found it.
  - Viewed index reconstruction.

Reviewed By: jungejason
Reviewers: jungejason, amckinley, tuomaspelkonen, aran
CC: aran, jungejason, epriestley
Differential Revision: 788
This commit is contained in:
epriestley 2011-08-07 15:14:23 -07:00
parent e35d72f489
commit b8e08f34f7
30 changed files with 244 additions and 103 deletions

View file

@ -35,6 +35,7 @@ return array(
// be 50x50px.
'user.default-profile-image-phid' => 'PHID-FILE-4d61229816cfe6f2b2a3',
// -- DarkConsole ----------------------------------------------------------- //
// DarkConsole is a administrative debugging/profiling tool built into
@ -67,6 +68,7 @@ return array(
'github.application-secret',
),
// -- MySQL --------------------------------------------------------------- //
// The username to use when connecting to MySQL.
@ -80,6 +82,7 @@ return array(
// (e.g., db.example.com:1234).
'mysql.host' => 'localhost',
// -- Email ----------------------------------------------------------------- //
// Some Phabricator tools send email notifications, e.g. when Differential
@ -322,6 +325,7 @@ return array(
// behalf, silencing the warning.
'phabricator.timezone' => null,
// -- Files ----------------------------------------------------------------- //
// Lists which uploaded file types may be viewed in the browser. If a file
@ -395,6 +399,18 @@ return array(
// fits within configured limits.
'storage.engine-selector' => 'PhabricatorDefaultFileStorageEngineSelector',
// -- Search ---------------------------------------------------------------- //
// Phabricator uses a search engine selector to choose which search engine
// to use when indexing and reconstructing documents, and when executing
// queries. You can override the engine selector to provide a new selector
// class which can select some custom engine you implement, if you want to
// store your documents in some search engine which does not have default
// support.
'search.engine-selector' => 'PhabricatorDefaultSearchEngineSelector',
// -- Differential ---------------------------------------------------------- //
'differential.revision-custom-detail-renderer' => null,
@ -440,6 +456,7 @@ return array(
// type.
'maniphest.custom-fields' => array(),
// -- Remarkup -------------------------------------------------------------- //
// If you enable this, linked YouTube videos will be embeded inline. This has

View file

@ -336,6 +336,7 @@ phutil_register_library_map(array(
'PhabricatorDaemonTimelineConsoleController' => 'applications/daemon/controller/timeline',
'PhabricatorDaemonTimelineEventController' => 'applications/daemon/controller/timelineevent',
'PhabricatorDefaultFileStorageEngineSelector' => 'applications/files/engineselector/default',
'PhabricatorDefaultSearchEngineSelector' => 'applications/search/selector/default',
'PhabricatorDifferenceEngine' => 'infrastructure/diff/engine',
'PhabricatorDirectoryCategory' => 'applications/directory/storage/category',
'PhabricatorDirectoryCategoryDeleteController' => 'applications/directory/controller/categorydelete',
@ -545,11 +546,12 @@ phutil_register_library_map(array(
'PhabricatorSearchDocumentField' => 'applications/search/storage/document/field',
'PhabricatorSearchDocumentIndexer' => 'applications/search/index/indexer/base',
'PhabricatorSearchDocumentRelationship' => 'applications/search/storage/document/relationship',
'PhabricatorSearchExecutor' => 'applications/search/execute/base',
'PhabricatorSearchEngine' => 'applications/search/engine/base',
'PhabricatorSearchEngineMySQL' => 'applications/search/engine/mysql',
'PhabricatorSearchEngineSelector' => 'applications/search/selector/base',
'PhabricatorSearchField' => 'applications/search/constants/field',
'PhabricatorSearchIndexController' => 'applications/search/controller/index',
'PhabricatorSearchManiphestIndexer' => 'applications/search/index/indexer/maniphest',
'PhabricatorSearchMySQLExecutor' => 'applications/search/execute/mysql',
'PhabricatorSearchPhrictionIndexer' => 'applications/search/index/indexer/phriction',
'PhabricatorSearchQuery' => 'applications/search/storage/query',
'PhabricatorSearchRelationship' => 'applications/search/constants/relationship',
@ -909,6 +911,7 @@ phutil_register_library_map(array(
'PhabricatorDaemonTimelineConsoleController' => 'PhabricatorDaemonController',
'PhabricatorDaemonTimelineEventController' => 'PhabricatorDaemonController',
'PhabricatorDefaultFileStorageEngineSelector' => 'PhabricatorFileStorageEngineSelector',
'PhabricatorDefaultSearchEngineSelector' => 'PhabricatorSearchEngineSelector',
'PhabricatorDirectoryCategory' => 'PhabricatorDirectoryDAO',
'PhabricatorDirectoryCategoryDeleteController' => 'PhabricatorDirectoryController',
'PhabricatorDirectoryCategoryEditController' => 'PhabricatorDirectoryController',
@ -1089,9 +1092,9 @@ phutil_register_library_map(array(
'PhabricatorSearchDocument' => 'PhabricatorSearchDAO',
'PhabricatorSearchDocumentField' => 'PhabricatorSearchDAO',
'PhabricatorSearchDocumentRelationship' => 'PhabricatorSearchDAO',
'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
'PhabricatorSearchIndexController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchManiphestIndexer' => 'PhabricatorSearchDocumentIndexer',
'PhabricatorSearchMySQLExecutor' => 'PhabricatorSearchExecutor',
'PhabricatorSearchPhrictionIndexer' => 'PhabricatorSearchDocumentIndexer',
'PhabricatorSearchQuery' => 'PhabricatorSearchDAO',
'PhabricatorSearchResultView' => 'AphrontView',

View file

@ -67,8 +67,8 @@ class DiffusionCommitListController extends DiffusionController {
),
phutil_escape_html($user->getUsername()));
$executor = new PhabricatorSearchMySQLExecutor();
$results = $executor->executeSearch($query);
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
$results = $engine->executeSearch($query);
$results = $pager->sliceResults($results);
$result_phids = ipull($results, 'phid');
$commit_table = self::createCommitTable($result_phids, $user);

View file

@ -10,7 +10,7 @@ phutil_require_module('phabricator', 'applications/diffusion/controller/base');
phutil_require_module('phabricator', 'applications/people/storage/user');
phutil_require_module('phabricator', 'applications/phid/constants');
phutil_require_module('phabricator', 'applications/phid/handle/data');
phutil_require_module('phabricator', 'applications/search/execute/mysql');
phutil_require_module('phabricator', 'applications/search/selector/base');
phutil_require_module('phabricator', 'applications/search/storage/query');
phutil_require_module('phabricator', 'view/control/pager');
phutil_require_module('phabricator', 'view/control/table');

View file

@ -32,8 +32,8 @@ class PhabricatorSearchIndexController extends PhabricatorSearchBaseController {
public function processRequest() {
$executor = new PhabricatorSearchMySQLExecutor();
$document = $executor->reconstructDocument($this->phid);
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
$document = $engine->reconstructDocument($this->phid);
if (!$document) {
return new Aphront404Response();
}

View file

@ -9,7 +9,7 @@
phutil_require_module('phabricator', 'aphront/response/404');
phutil_require_module('phabricator', 'applications/phid/handle/data');
phutil_require_module('phabricator', 'applications/search/controller/base');
phutil_require_module('phabricator', 'applications/search/execute/mysql');
phutil_require_module('phabricator', 'applications/search/selector/base');
phutil_require_module('phabricator', 'view/control/table');
phutil_require_module('phabricator', 'view/layout/panel');

View file

@ -168,8 +168,8 @@ class PhabricatorSearchController extends PhabricatorSearchBaseController {
$query->setParameter('limit', $limit + 1);
$query->setParameter('offset', $pager->getOffset());
$executor = new PhabricatorSearchMySQLExecutor();
$results = $executor->executeSearch($query);
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
$results = $engine->executeSearch($query);
$results = ipull($results, 'phid');
$results = $pager->sliceResults($results);

View file

@ -11,7 +11,7 @@ phutil_require_module('phabricator', 'aphront/response/redirect');
phutil_require_module('phabricator', 'applications/phid/constants');
phutil_require_module('phabricator', 'applications/phid/handle/data');
phutil_require_module('phabricator', 'applications/search/controller/base');
phutil_require_module('phabricator', 'applications/search/execute/mysql');
phutil_require_module('phabricator', 'applications/search/selector/base');
phutil_require_module('phabricator', 'applications/search/storage/query');
phutil_require_module('phabricator', 'applications/search/view/searchresult');
phutil_require_module('phabricator', 'infrastructure/celerity/api');

View file

@ -0,0 +1,57 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Base class for Phabricator search engine providers. Each engine must offer
* three capabilities: indexing, searching, and reconstruction (this can be
* stubbed out if an engine can't reasonably do it, it is used for debugging).
*
* @group search
*/
abstract class PhabricatorSearchEngine {
/**
* Update the index for an abstract document.
*
* @param PhabricatorSearchAbstractDocument Document to update.
* @return void
*/
abstract public function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $document);
/**
* Reconstruct the document for a given PHID. This is used for debugging
* and does not need to be perfect if it is unreasonable to implement it.
*
* @param phid Document PHID to reconstruct.
* @return PhabricatorSearchAbstractDocument Abstract document.
*/
abstract public function reconstructDocument($phid);
/**
* Execute a search query.
*
* @param PhabricatorSearchQuery A query to execute.
* @return list A list of matching PHIDs.
*/
abstract public function executeSearch(PhabricatorSearchQuery $query);
}

View file

@ -7,4 +7,4 @@
phutil_require_source('PhabricatorSearchExecutor.php');
phutil_require_source('PhabricatorSearchEngine.php');

View file

@ -16,7 +16,77 @@
* limitations under the License.
*/
class PhabricatorSearchMySQLExecutor extends PhabricatorSearchExecutor {
final class PhabricatorSearchEngineMySQL extends PhabricatorSearchEngine {
public function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $doc) {
$phid = $doc->getPHID();
if (!$phid) {
throw new Exception("Document has no PHID!");
}
$store = new PhabricatorSearchDocument();
$store->setPHID($doc->getPHID());
$store->setDocumentType($doc->getDocumentType());
$store->setDocumentTitle($doc->getDocumentTitle());
$store->setDocumentCreated($doc->getDocumentCreated());
$store->setDocumentModified($doc->getDocumentModified());
$store->replace();
$conn_w = $store->establishConnection('w');
$field_dao = new PhabricatorSearchDocumentField();
queryfx(
$conn_w,
'DELETE FROM %T WHERE phid = %s',
$field_dao->getTableName(),
$phid);
foreach ($doc->getFieldData() as $field) {
list($ftype, $corpus, $aux_phid) = $field;
queryfx(
$conn_w,
'INSERT INTO %T (phid, phidType, field, auxPHID, corpus) '.
' VALUES (%s, %s, %s, %ns, %s)',
$field_dao->getTableName(),
$phid,
$doc->getDocumentType(),
$ftype,
$aux_phid,
$corpus);
}
$sql = array();
foreach ($doc->getRelationshipData() as $relationship) {
list($rtype, $to_phid, $to_type, $time) = $relationship;
$sql[] = qsprintf(
$conn_w,
'(%s, %s, %s, %s, %d)',
$phid,
$to_phid,
$rtype,
$to_type,
$time);
}
$rship_dao = new PhabricatorSearchDocumentRelationship();
queryfx(
$conn_w,
'DELETE FROM %T WHERE phid = %s',
$rship_dao->getTableName(),
$phid);
if ($sql) {
queryfx(
$conn_w,
'INSERT INTO %T'.
' (phid, relatedPHID, relation, relatedType, relatedTime) '.
' VALUES %Q',
$rship_dao->getTableName(),
implode(', ', $sql));
}
}
/**
* Rebuild the PhabricatorSearchAbstractDocument that was used to index

View file

@ -7,7 +7,7 @@
phutil_require_module('phabricator', 'applications/search/constants/relationship');
phutil_require_module('phabricator', 'applications/search/execute/base');
phutil_require_module('phabricator', 'applications/search/engine/base');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'applications/search/storage/document/document');
phutil_require_module('phabricator', 'applications/search/storage/document/field');
@ -18,4 +18,4 @@ phutil_require_module('phabricator', 'storage/queryfx');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorSearchMySQLExecutor.php');
phutil_require_source('PhabricatorSearchEngineMySQL.php');

View file

@ -16,4 +16,13 @@
* limitations under the License.
*/
abstract class PhabricatorSearchDocumentIndexer { }
abstract class PhabricatorSearchDocumentIndexer {
// TODO: Make this whole class tree concrete?
final protected static function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $document) {
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
$engine->reindexAbstractDocument($document);
}
}

View file

@ -6,5 +6,7 @@
phutil_require_module('phabricator', 'applications/search/selector/base');
phutil_require_source('PhabricatorSearchDocumentIndexer.php');

View file

@ -107,6 +107,6 @@ class PhabricatorSearchDifferentialIndexer
$rev->getDateModified()); // Bogus timestamp.
}
PhabricatorSearchDocument::reindexAbstractDocument($doc);
self::reindexAbstractDocument($doc);
}
}

View file

@ -14,7 +14,6 @@ phutil_require_module('phabricator', 'applications/search/constants/field');
phutil_require_module('phabricator', 'applications/search/constants/relationship');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'applications/search/index/indexer/base');
phutil_require_module('phabricator', 'applications/search/storage/document/document');
phutil_require_module('phutil', 'utils');

View file

@ -127,6 +127,6 @@ class PhabricatorSearchManiphestIndexer
$time);
}
PhabricatorSearchDocument::reindexAbstractDocument($doc);
self::reindexAbstractDocument($doc);
}
}

View file

@ -16,7 +16,6 @@ phutil_require_module('phabricator', 'applications/search/constants/field');
phutil_require_module('phabricator', 'applications/search/constants/relationship');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'applications/search/index/indexer/base');
phutil_require_module('phabricator', 'applications/search/storage/document/document');
phutil_require_module('phutil', 'utils');

View file

@ -41,6 +41,6 @@ class PhabricatorSearchPhrictionIndexer
PhabricatorPHIDConstants::PHID_TYPE_USER,
$content->getDateCreated());
PhabricatorSearchDocument::reindexAbstractDocument($doc);
self::reindexAbstractDocument($doc);
}
}

View file

@ -11,7 +11,6 @@ phutil_require_module('phabricator', 'applications/search/constants/field');
phutil_require_module('phabricator', 'applications/search/constants/relationship');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'applications/search/index/indexer/base');
phutil_require_module('phabricator', 'applications/search/storage/document/document');
phutil_require_source('PhabricatorSearchPhrictionIndexer.php');

View file

@ -63,7 +63,7 @@ class PhabricatorSearchCommitIndexer
PhabricatorPHIDConstants::PHID_TYPE_REPO,
$date_created);
PhabricatorSearchDocument::reindexAbstractDocument($doc);
self::reindexAbstractDocument($doc);
}
}

View file

@ -13,7 +13,6 @@ phutil_require_module('phabricator', 'applications/search/constants/field');
phutil_require_module('phabricator', 'applications/search/constants/relationship');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'applications/search/index/indexer/base');
phutil_require_module('phabricator', 'applications/search/storage/document/document');
phutil_require_module('phutil', 'utils');

View file

@ -30,6 +30,6 @@ class PhabricatorSearchUserIndexer
// TODO: Index the blurbs from their profile or something? Probably not
// actually useful...
PhabricatorSearchDocument::reindexAbstractDocument($doc);
self::reindexAbstractDocument($doc);
}
}

View file

@ -9,7 +9,6 @@
phutil_require_module('phabricator', 'applications/phid/constants');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'applications/search/index/indexer/base');
phutil_require_module('phabricator', 'applications/search/storage/document/document');
phutil_require_source('PhabricatorSearchUserIndexer.php');

View file

@ -0,0 +1,32 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
abstract class PhabricatorSearchEngineSelector {
final public function __construct() {
// <empty>
}
abstract public function newEngine();
final public static function newSelector() {
$class = PhabricatorEnv::getEnvConfig('search.engine-selector');
return newv($class, array());
}
}

View file

@ -0,0 +1,14 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'infrastructure/env');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorSearchEngineSelector.php');

View file

@ -16,7 +16,10 @@
* limitations under the License.
*/
abstract class PhabricatorSearchExecutor {
final class PhabricatorDefaultSearchEngineSelector
extends PhabricatorSearchEngineSelector {
public function newEngine() {
return new PhabricatorSearchEngineMySQL();
}
}

View file

@ -0,0 +1,13 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/search/engine/mysql');
phutil_require_module('phabricator', 'applications/search/selector/base');
phutil_require_source('PhabricatorDefaultSearchEngineSelector.php');

View file

@ -35,74 +35,4 @@ class PhabricatorSearchDocument extends PhabricatorSearchDAO {
return 'phid';
}
public static function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $doc) {
$phid = $doc->getPHID();
if (!$phid) {
throw new Exception("Document has no PHID!");
}
$store = new PhabricatorSearchDocument();
$store->setPHID($doc->getPHID());
$store->setDocumentType($doc->getDocumentType());
$store->setDocumentTitle($doc->getDocumentTitle());
$store->setDocumentCreated($doc->getDocumentCreated());
$store->setDocumentModified($doc->getDocumentModified());
$store->replace();
$conn_w = $store->establishConnection('w');
$field_dao = new PhabricatorSearchDocumentField();
queryfx(
$conn_w,
'DELETE FROM %T WHERE phid = %s',
$field_dao->getTableName(),
$phid);
foreach ($doc->getFieldData() as $field) {
list($ftype, $corpus, $aux_phid) = $field;
queryfx(
$conn_w,
'INSERT INTO %T (phid, phidType, field, auxPHID, corpus) '.
' VALUES (%s, %s, %s, %ns, %s)',
$field_dao->getTableName(),
$phid,
$doc->getDocumentType(),
$ftype,
$aux_phid,
$corpus);
}
$sql = array();
foreach ($doc->getRelationshipData() as $relationship) {
list($rtype, $to_phid, $to_type, $time) = $relationship;
$sql[] = qsprintf(
$conn_w,
'(%s, %s, %s, %s, %d)',
$phid,
$to_phid,
$rtype,
$to_type,
$time);
}
$rship_dao = new PhabricatorSearchDocumentRelationship();
queryfx(
$conn_w,
'DELETE FROM %T WHERE phid = %s',
$rship_dao->getTableName(),
$phid);
if ($sql) {
queryfx(
$conn_w,
'INSERT INTO %T'.
' (phid, relatedPHID, relation, relatedType, relatedTime) '.
' VALUES %Q',
$rship_dao->getTableName(),
implode(', ', $sql));
}
}
}

View file

@ -7,10 +7,6 @@
phutil_require_module('phabricator', 'applications/search/storage/base');
phutil_require_module('phabricator', 'applications/search/storage/document/field');
phutil_require_module('phabricator', 'applications/search/storage/document/relationship');
phutil_require_module('phabricator', 'storage/qsprintf');
phutil_require_module('phabricator', 'storage/queryfx');
phutil_require_source('PhabricatorSearchDocument.php');