revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00
Code Issues Releases Wiki Activity

Use the Ferret engine fulltext document table to drive auxiliary fulltext constraints

Browse source 
Summary:
Ref T12819. I started trying to get individual engines to drive these constraints (e.g., `ManiphestTaskQuery` can do most of the work) but this is a big pain, especially since most engines don't support "any owner" or "no owner", and not everything has an owner, and so on and so on. Going down this path would have meant a huge pile of stub functions everywhere, I think.

Instead, drive these through the main engine using the fulltext document table, which already has everything we need to apply these constraints in a uniform way.

Also tweak some parts of query construction and result ordering.

Test Plan: Searched for documents by author, owner, unowned, any owner, tags, subscribers, fulltext in global search. Got sensible results without any application-specific code.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T12819

Differential Revision: https://secure.phabricator.com/D18550
This commit is contained in:
epriestley 2017-09-07 08:33:20 -07:00
parent 3720b28c6c
commit 8059db894d
7 changed files with 171 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/applications/differential/search/DifferentialRevisionFerretEngine.php
View file

@ -15,7 +15,7 @@ final class DifferentialRevisionFerretEngine
return new DifferentialRevisionFerretField();
}
protected function newSearchEngine() {
public function newSearchEngine() {
return new DifferentialRevisionSearchEngine();
}

2
src/applications/maniphest/search/ManiphestTaskFerretEngine.php
View file

@ -15,7 +15,7 @@ final class ManiphestTaskFerretEngine
return new ManiphestTaskFerretField();
}
protected function newSearchEngine() {
public function newSearchEngine() {
return new ManiphestTaskSearchEngine();
}

32
src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
View file

@ -23,11 +23,37 @@ final class PhabricatorFerretFulltextEngineExtension
$phid = $document->getPHID();
$engine = $object->newFerretEngine();
$is_closed = 0;
$author_phid = null;
$owner_phid = null;
foreach ($document->getRelationshipData() as $relationship) {
list($related_type, $related_phid) = $relationship;
switch ($related_type) {
case PhabricatorSearchRelationship::RELATIONSHIP_OPEN:
$is_closed = 0;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_CLOSED:
$is_closed = 1;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_OWNER:
$owner_phid = $related_phid;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED:
$owner_phid = null;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR:
$author_phid = $related_phid;
break;
}
}
$ferret_document = $engine->newDocumentObject()
->setObjectPHID($phid)
->setIsClosed(0)
->setEpochCreated(0)
->setEpochModified(0);
->setIsClosed($is_closed)
->setEpochCreated($document->getDocumentCreated())
->setEpochModified($document->getDocumentModified())
->setAuthorPHID($author_phid)
->setOwnerPHID($owner_phid);
$stemmer = $engine->newStemmer();

12
src/applications/search/ferret/PhabricatorFerretDocument.php
View file

@ -27,6 +27,18 @@ abstract class PhabricatorFerretDocument
'columns' => array('objectPHID'),
'unique' => true,
),
'key_author' => array(
'columns' => array('authorPHID'),
),
'key_owner' => array(
'columns' => array('ownerPHID'),
),
'key_created' => array(
'columns' => array('epochCreated'),
),
'key_modified' => array(
'columns' => array('epochModified'),
),
),
) + parent::getConfiguration();
}

56
src/applications/search/ferret/PhabricatorFerretEngine.php
View file

@ -5,7 +5,7 @@ abstract class PhabricatorFerretEngine extends Phobject {
abstract public function newNgramsObject();
abstract public function newDocumentObject();
abstract public function newFieldObject();
abstract protected function newSearchEngine();
abstract public function newSearchEngine();
public function getDefaultFunctionKey() {
return 'all';
@ -70,60 +70,6 @@ abstract class PhabricatorFerretEngine extends Phobject {
return new PhutilSearchStemmer();
}
public function newConfiguredFulltextQuery(
$object,
PhabricatorSavedQuery $query,
PhabricatorUser $viewer) {
$local_query = new PhabricatorSavedQuery();
$local_query->setParameter('query', $query->getParameter('query'));
// TODO: Modularize this piece.
$project_phids = $query->getParameter('projectPHIDs');
if ($project_phids) {
$local_query->setParameter('projectPHIDs', $project_phids);
}
$subscriber_phids = $query->getParameter('subscriberPHIDs');
if ($subscriber_phids) {
$local_query->setParameter('subscriberPHIDs', $subscriber_phids);
}
$author_phids = $query->getParameter('authorPHIDs');
if ($author_phids) {
$local_query->setParameter('authorPHIDs', $author_phids);
}
$owner_phids = $query->getParameter('ownerPHIDs');
if ($owner_phids) {
$local_query->setParameter('ownerPHIDs', $owner_phids);
}
$rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN;
$rel_closed = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED;
$statuses = $query->getParameter('statuses');
if ($statuses) {
$statuses = array_fuse($statuses);
if (count($statuses) == 1) {
if (isset($statuses[$rel_open])) {
$local_query->setParameter('statuses', array('open()'));
}
if (isset($statuses[$rel_closed])) {
$local_query->setParameter('statuses', array('closed()'));
}
}
}
$search_engine = $this->newSearchEngine()
->setViewer($viewer);
$engine_query = $search_engine->buildQueryFromSavedQuery($local_query)
->setViewer($viewer);
return $engine_query;
}
public function tokenizeString($value) {
$value = trim($value, ' ');
$value = preg_split('/ +/', $value);

30
src/applications/search/fulltextstorage/PhabricatorFerretFulltextStorageEngine.php
View file

@ -47,6 +47,8 @@ final class PhabricatorFerretFulltextStorageEngine
$offset = (int)$query->getParameter('offset', 0);
$limit = (int)$query->getParameter('limit', 25);
// NOTE: For now, it's okay to query with the omnipotent viewer here
// because we're just returning PHIDs which we'll filter later.
$viewer = PhabricatorUser::getOmnipotentUser();
$type_results = array();
@ -54,19 +56,31 @@ final class PhabricatorFerretFulltextStorageEngine
$engine = $spec['engine'];
$object = $spec['object'];
// NOTE: For now, it's okay to query with the omnipotent viewer here
// because we're just returning PHIDs which we'll filter later.
$local_query = new PhabricatorSavedQuery();
$local_query->setParameter('query', $query->getParameter('query'));
$type_query = $engine->newConfiguredFulltextQuery(
$object,
$query,
$viewer);
$project_phids = $query->getParameter('projectPHIDs');
if ($project_phids) {
$local_query->setParameter('projectPHIDs', $project_phids);
}
$type_query
$subscriber_phids = $query->getParameter('subscriberPHIDs');
if ($subscriber_phids) {
$local_query->setParameter('subscriberPHIDs', $subscriber_phids);
}
$search_engine = $engine->newSearchEngine()
->setViewer($viewer);
$engine_query = $search_engine->buildQueryFromSavedQuery($local_query)
->setViewer($viewer);
$engine_query
->withFerretQuery($engine, $query)
->setOrder('relevance')
->setLimit($offset + $limit);
$results = $type_query->execute();
$results = $engine_query->execute();
$results = mpull($results, null, 'getPHID');
$type_results[$type] = $results;
}

114
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
View file

@ -28,8 +28,9 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
private $spaceIsArchived;
private $ngrams = array();
private $ferretEngine;
private $ferretTokens;
private $ferretTables;
private $ferretTokens = array();
private $ferretTables = array();
private $ferretQuery;
protected function getPageCursors(array $page) {
return array(
@ -772,7 +773,7 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
if ($this->supportsFerretEngine()) {
$orders['relevance'] = array(
'vector' => array('rank', 'id'),
'vector' => array('rank', 'fulltext-modified', 'id'),
'name' => pht('Relevence'),
);
}
@ -975,6 +976,16 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
'column' => '_ft_rank',
'type' => 'int',
);
$columns['fulltext-created'] = array(
'table' => 'ft_doc',
'column' => 'epochCreated',
'type' => 'int',
);
$columns['fulltext-modified'] = array(
'table' => 'ft_doc',
'column' => 'epochModified',
'type' => 'int',
);
}
$cache->setKey($cache_key, $columns);
@ -1406,6 +1417,22 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
return ($object instanceof PhabricatorFerretInterface);
}
public function withFerretQuery(
PhabricatorFerretEngine $engine,
PhabricatorSavedQuery $query) {
if (!$this->supportsFerretEngine()) {
throw new Exception(
pht(
'Query ("%s") does not support the Ferret fulltext engine.',
get_class($this)));
}
$this->ferretEngine = $engine;
$this->ferretQuery = $query;
return $this;
}
public function withFerretConstraint(
PhabricatorFerretEngine $engine,
@ -1538,10 +1565,10 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
$table_alias,
$stem_value);
}
$parts[] = '0';
}
$parts[] = '0';
$select[] = qsprintf(
$conn,
'%Q _ft_rank',
@ -1646,7 +1673,7 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
$joins = array();
$joins[] = qsprintf(
$conn,
'JOIN %T ftdoc ON ftdoc.objectPHID = %Q',
'JOIN %T ft_doc ON ft_doc.objectPHID = %Q',
$document_table->getTableName(),
$phid_column);
@ -1655,11 +1682,11 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
$table = $spec['table'];
$ngram = $spec['ngram'];
$alias = 'ft'.$idx++;
$alias = 'ftngram_'.$idx++;
$joins[] = qsprintf(
$conn,
'JOIN %T %T ON %T.documentID = ftdoc.id AND %T.ngram = %s',
'JOIN %T %T ON %T.documentID = ft_doc.id AND %T.ngram = %s',
$table,
$alias,
$alias,
@ -1672,7 +1699,7 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
$joins[] = qsprintf(
$conn,
'JOIN %T %T ON ftdoc.id = %T.documentID
'JOIN %T %T ON ft_doc.id = %T.documentID
AND %T.fieldKey = %s',
$field_table->getTableName(),
$alias,
@ -1801,6 +1828,75 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
}
}
if ($this->ferretQuery) {
$query = $this->ferretQuery;
$author_phids = $query->getParameter('authorPHIDs');
if ($author_phids) {
$where[] = qsprintf(
$conn,
'ft_doc.authorPHID IN (%Ls)',
$author_phids);
}
$with_unowned = $query->getParameter('withUnowned');
$with_any = $query->getParameter('withAnyOwner');
if ($with_any && $with_unowned) {
throw new PhabricatorEmptyQueryException(
pht(
'This query matches only unowned documents owned by anyone, '.
'which is impossible.'));
}
$owner_phids = $query->getParameter('ownerPHIDs');
if ($owner_phids && !$with_any) {
if ($with_unowned) {
$where[] = qsprintf(
$conn,
'ft_doc.ownerPHID IN (%Ls) OR ft_doc.ownerPHID IS NULL',
$owner_phids);
} else {
$where[] = qsprintf(
$conn,
'ft_doc.ownerPHID IN (%Ls)',
$owner_phids);
}
} else if ($with_unowned) {
$where[] = qsprintf(
$conn,
'ft_doc.ownerPHID IS NULL');
}
if ($with_any) {
$where[] = qsprintf(
$conn,
'ft_doc.ownerPHID IS NOT NULL');
}
$rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN;
$statuses = $query->getParameter('statuses');
$is_closed = null;
if ($statuses) {
$statuses = array_fuse($statuses);
if (count($statuses) == 1) {
if (isset($statuses[$rel_open])) {
$is_closed = 0;
} else {
$is_closed = 1;
}
}
}
if ($is_closed !== null) {
$where[] = qsprintf(
$conn,
'ft_doc.isClosed = %d',
$is_closed);
}
}
return $where;
}