revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-26 16:52:41 +01:00
Code Issues Releases Wiki Activity

Improve elasticsearch

Browse source 
Summary: I thought that this will be fun but the elasticsearch API is horrible and the documentation is poor.

Test Plan:
Search for:

- string
- author
- author, owner
- string, author
- open
- string, open, author
- string, exclude
- several authors, several owners
- nothing
- probably all other combinations

Normally, such an exhaustive test plan wouldn't be required but each combination requires a completely different query.

Reviewers: epriestley, jungejason

Reviewed By: epriestley

CC: aran, Koolvin, btrahan

Differential Revision: https://secure.phabricator.com/D2298
This commit is contained in:
vrana 2012-04-20 17:09:30 -07:00
parent 7b334f37bd
commit 44bbdec9ac
8 changed files with 126 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/applications/search/controller/search/PhabricatorSearchController.php
View file

@ -113,16 +113,9 @@ final class PhabricatorSearchController
}
}
$more = PhabricatorEnv::getEnvConfig('search.more-document-types', array());
$options = array(
'' => 'All Documents',
PhabricatorPHIDConstants::PHID_TYPE_DREV => 'Differential Revisions',
PhabricatorPHIDConstants::PHID_TYPE_CMIT => 'Repository Commits',
PhabricatorPHIDConstants::PHID_TYPE_TASK => 'Maniphest Tasks',
PhabricatorPHIDConstants::PHID_TYPE_WIKI => 'Phriction Documents',
PhabricatorPHIDConstants::PHID_TYPE_USER => 'Phabricator Users',
) + $more;
) + PhabricatorSearchAbstractDocument::getSupportedTypes();
$status_options = array(
0 => 'Open and Closed Documents',

2
src/applications/search/controller/search/__init__.php
View file

@ -18,11 +18,11 @@ phutil_require_module('phabricator', 'applications/repository/storage/repository
phutil_require_module('phabricator', 'applications/search/constants/scope');
phutil_require_module('phabricator', 'applications/search/controller/base');
phutil_require_module('phabricator', 'applications/search/engine/jumpnav');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'applications/search/selector/base');
phutil_require_module('phabricator', 'applications/search/storage/query');
phutil_require_module('phabricator', 'applications/search/view/searchresult');
phutil_require_module('phabricator', 'infrastructure/celerity/api');
phutil_require_module('phabricator', 'infrastructure/env');
phutil_require_module('phabricator', 'view/control/pager');
phutil_require_module('phabricator', 'view/form/base');
phutil_require_module('phabricator', 'view/form/control/select');

149
src/applications/search/engine/elastic/PhabricatorSearchEngineElastic.php
View file

@ -1,7 +1,7 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -23,23 +23,19 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
$type = $doc->getDocumentType();
$phid = $doc->getPHID();
$handle = PhabricatorObjectHandleData::loadOneHandle($phid);
$spec = array(
'phid' => $phid,
'type' => $type,
'title' => $doc->getDocumentTitle(),
'dateCreated' => date('c', $doc->getDocumentCreated()),
'dateModified' => date('c', $doc->getDocumentModified()),
'url' => PhabricatorEnv::getProductionURI($handle->getURI()),
'dateCreated' => $doc->getDocumentCreated(),
'_timestamp' => $doc->getDocumentModified(),
'field' => array(),
'relationship' => array(),
);
foreach ($doc->getFieldData() as $field) {
list($ftype, $corpus, $aux_phid) = $field;
$spec['field'][$ftype][] = array(
'corpus' => $corpus,
'aux' => $aux_phid,
);
$spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field);
}
foreach ($doc->getRelationshipData() as $relationship) {
@ -47,7 +43,7 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
$spec['relationship'][$rtype][] = array(
'phid' => $to_phid,
'phidType' => $to_type,
'when' => date('c', $time),
'when' => $time,
);
}
@ -58,39 +54,25 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
}
public function reconstructDocument($phid) {
$type = phid_get_type($phid);
$response = $this->executeRequest(
'/phabricator/_search',
array(
'query' => array(
'ids' => array(
'values' => array(
$phid,
),
),
),
),
$is_write = false);
$response = $this->executeRequest("/phabricator/{$type}/{$phid}", array());
$hit = $response['hits']['hits'][0]['_source'];
if (!$hit) {
if (empty($response['exists'])) {
return null;
}
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($hit['phid']);
$doc->setDocumentType($hit['type']);
$doc->setDocumentTitle($hit['title']);
$doc->setDocumentCreated(strtotime($hit['dateCreated']));
$doc->setDocumentModified(strtotime($hit['dateModified']));
$hit = $response['_source'];
foreach ($hit['field'] as $ftype => $fdefs) {
foreach ($fdefs as $fdef) {
$doc->addField(
$ftype,
$fdef['corpus'],
$fdef['aux']);
}
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($phid);
$doc->setDocumentType($response['_type']);
$doc->setDocumentTitle($hit['title']);
$doc->setDocumentCreated($hit['dateCreated']);
$doc->setDocumentModified($hit['_timestamp']);
foreach ($hit['field'] as $fdef) {
$doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']);
}
foreach ($hit['relationship'] as $rtype => $rships) {
@ -99,7 +81,7 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
$rtype,
$rship['phid'],
$rship['phidType'],
strtotime($rship['when']));
$rship['when']);
}
}
@ -108,35 +90,96 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
public function executeSearch(PhabricatorSearchQuery $query) {
$spec = array(
'text' => array(
'_all' => $query->getQuery(),
$spec = array();
$filter = array();
if ($query->getQuery()) {
$spec[] = array(
'field' => array(
'field.corpus' => $query->getQuery(),
),
);
}
$exclude = $query->getParameter('exclude');
if ($exclude) {
$filter[] = array(
'not' => array(
'ids' => array(
'values' => array($exclude),
),
),
);
}
$type = $query->getParameter('type');
if ($type) {
$uri = "/phabricator/{$type}/_search";
} else {
$uri = "/phabricator/_search";
// Don't use '/phabricator/_search' for the case that there is something
// else in the index (for example if 'phabricator' is only an alias to
// some bigger index).
$types = PhabricatorSearchAbstractDocument::getSupportedTypes();
$uri = '/phabricator/' . implode(',', array_keys($types)) . '/_search';
}
$response = $this->executeRequest(
$uri,
array(
'query' => $spec,
$rel_mapping = array(
'author' => PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR,
'open' => PhabricatorSearchRelationship::RELATIONSHIP_OPEN,
'owner' => PhabricatorSearchRelationship::RELATIONSHIP_OWNER,
'project' => PhabricatorSearchRelationship::RELATIONSHIP_PROJECT,
'repository' => PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY,
);
foreach ($rel_mapping as $name => $field) {
$param = $query->getParameter($name);
if (is_array($param)) {
$should = array();
foreach ($param as $val) {
$should[] = array(
'text' => array(
"relationship.{$field}.phid" => array(
'query' => $val,
'type' => 'phrase',
),
$is_write = false);
$phids = array();
foreach ($response['hits']['hits'] as $hit) {
$phids[] = $hit['_id'];
),
);
}
$spec[] = array('bool' => array('should' => $should));
} else if ($param) {
$filter[] = array(
'exists' => array(
'field' => "relationship.{$field}.phid",
),
);
}
}
if ($spec) {
$spec = array('query' => array('bool' => array('must' => $spec)));
}
if ($filter) {
$filter = array('filter' => array('and' => $filter));
if ($spec) {
$spec = array(
'query' => array(
'filtered' => $spec + $filter,
),
);
} else {
$spec = $filter;
}
}
$spec['from'] = (int)$query->getParameter('offset', 0);
$spec['size'] = (int)$query->getParameter('limit', 0);
$response = $this->executeRequest($uri, $spec);
$phids = ipull($response['hits']['hits'], '_id');
return $phids;
}
private function executeRequest($path, array $data, $is_write) {
private function executeRequest($path, array $data, $is_write = false) {
$uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
$uri = new PhutilURI($uri);
$data = json_encode($data);

4
src/applications/search/engine/elastic/__init__.php
View file

@ -6,6 +6,9 @@
phutil_require_module('phabricator', 'applications/phid/handle/data');
phutil_require_module('phabricator', 'applications/phid/utils');
phutil_require_module('phabricator', 'applications/search/constants/relationship');
phutil_require_module('phabricator', 'applications/search/engine/base');
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
phutil_require_module('phabricator', 'infrastructure/env');
@ -13,6 +16,7 @@ phutil_require_module('phabricator', 'infrastructure/env');
phutil_require_module('phutil', 'future/http/http');
phutil_require_module('phutil', 'future/http/https');
phutil_require_module('phutil', 'parser/uri');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorSearchEngineElastic.php');

7
src/applications/search/engine/mysql/PhabricatorSearchEngineMySQL.php
View file

@ -305,7 +305,12 @@ final class PhabricatorSearchEngineMySQL extends PhabricatorSearchEngine {
return ipull($hits, 'phid');
}
protected function joinRelationship($conn, $query, $field, $type) {
protected function joinRelationship(
AphrontDatabaseConnection $conn,
PhabricatorSearchQuery $query,
$field,
$type) {
$phids = $query->getParameter($field, array());
if (!$phids) {
return null;

13
src/applications/search/index/abstractdocument/PhabricatorSearchAbstractDocument.php
View file

@ -1,7 +1,7 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -29,6 +29,17 @@ final class PhabricatorSearchAbstractDocument {
private $fields = array();
private $relationships = array();
public static function getSupportedTypes() {
$more = PhabricatorEnv::getEnvConfig('search.more-document-types', array());
return array(
PhabricatorPHIDConstants::PHID_TYPE_DREV => 'Differential Revisions',
PhabricatorPHIDConstants::PHID_TYPE_CMIT => 'Repository Commits',
PhabricatorPHIDConstants::PHID_TYPE_TASK => 'Maniphest Tasks',
PhabricatorPHIDConstants::PHID_TYPE_WIKI => 'Phriction Documents',
PhabricatorPHIDConstants::PHID_TYPE_USER => 'Phabricator Users',
) + $more;
}
public function setPHID($phid) {
$this->phid = $phid;
return $this;

2
src/applications/search/index/abstractdocument/__init__.php
View file

@ -6,7 +6,9 @@
phutil_require_module('phabricator', 'applications/phid/constants');
phutil_require_module('phabricator', 'applications/search/constants/field');
phutil_require_module('phabricator', 'infrastructure/env');
phutil_require_source('PhabricatorSearchAbstractDocument.php');

2
src/applications/search/index/indexer/user/PhabricatorSearchUserIndexer.php
View file

@ -26,7 +26,7 @@ final class PhabricatorSearchUserIndexer
$doc = new PhabricatorSearchAbstractDocument();
$doc->setPHID($user->getPHID());
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_USER);
$doc->setDocumentTitle($user->getUserName().'('.$user->getRealName().')');
$doc->setDocumentTitle($user->getUserName().' ('.$user->getRealName().')');
$doc->setDocumentCreated($user->getDateCreated());
$doc->setDocumentModified($user->getDateModified());