1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-26 00:32:42 +01:00

Support multiple fulltext search clusters with 'cluster.search' config

Summary:
The goal is to make fulltext search back-ends more extensible, configurable and robust.

When this is finished it will be possible to have multiple search storage back-ends and
potentially multiple instances of each.

Individual instances can be configured with roles such as 'read', 'write' which control
which hosts will receive writes to the index and which hosts will respond to queries.

These two roles make it possible to have any combination of:

* read-only
* write-only
* read-write
* disabled

This 'roles' mechanism is extensible to add new roles should that be needed in the future.

In addition to supporting multiple elasticsearch and mysql search instances, this refactors
the connection health monitoring infrastructure from PhabricatorDatabaseHealthRecord and
utilizes the same system for monitoring the health of elasticsearch nodes. This will
allow Wikimedia's phabricator to be redundant across data centers (mysql already is,
elasticsearch should be as well).

The real-world use-case I have in mind here is writing to two indexes (two elasticsearch clusters
in different data centers) but reading from only one. Then toggling the 'read' property when
we want to migrate to the other data center (and when we migrate from elasticsearch 2.x to 5.x)

Hopefully this is useful in the upstream as well.

Remaining TODO:

* test cases
* documentation

Test Plan:
(WARNING) This will most likely require the elasticsearch index to be deleted and re-created due to schema changes.

Tested with elasticsearch versions 2.4 and 5.2 using the following config:

```lang=json
  "cluster.search": [
    {
      "type": "elasticsearch",
      "hosts": [
        {
          "host": "localhost",
          "roles": { "read": true, "write": true }
        }
      ],
      "port": 9200,
      "protocol": "http",
      "path": "/phabricator",
      "version": 5
    },
    {
      "type": "mysql",
      "roles": { "write": true }
     }
  ]

Also deployed the same changes to Wikimedia's production Phabricator instance without any issues whatsoever.
```

Reviewers: epriestley, #blessed_reviewers

Reviewed By: epriestley, #blessed_reviewers

Subscribers: Korvin, epriestley

Tags: #elasticsearch, #clusters, #wikimedia

Differential Revision: https://secure.phabricator.com/D17384
This commit is contained in:
Mukunda Modell 2017-03-26 08:16:47 +00:00 committed by 20after4
parent a41d158490
commit e41c25de50
36 changed files with 1411 additions and 378 deletions

View file

@ -1,7 +1,15 @@
<?php
$search_engine = PhabricatorFulltextStorageEngine::loadEngine();
$use_mysql = ($search_engine instanceof PhabricatorMySQLFulltextStorageEngine);
$use_mysql = false;
$services = PhabricatorSearchService::getAllServices();
foreach ($services as $service) {
$engine = $service->getEngine();
if ($engine instanceof PhabricatorMySQLFulltextStorageEngine) {
$use_mysql = true;
}
}
if ($use_mysql) {
$field = new PhabricatorSearchDocumentField();

View file

@ -2259,12 +2259,15 @@ phutil_register_library_map(array(
'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php',
'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php',
'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php',
'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php',
'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php',
'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php',
'PhabricatorClusterException' => 'infrastructure/cluster/exception/PhabricatorClusterException.php',
'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php',
'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php',
'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php',
'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php',
'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php',
'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php',
'PhabricatorClusterStrandedException' => 'infrastructure/cluster/exception/PhabricatorClusterStrandedException.php',
'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
@ -2310,6 +2313,7 @@ phutil_register_library_map(array(
'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php',
'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php',
'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php',
'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php',
'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php',
'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
@ -2543,7 +2547,6 @@ phutil_register_library_map(array(
'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php',
'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php',
'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php',
'PhabricatorDatabaseRefParser' => 'infrastructure/cluster/PhabricatorDatabaseRefParser.php',
'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
@ -2651,6 +2654,8 @@ phutil_register_library_map(array(
'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php',
'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php',
'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php',
'PhabricatorElasticSearchHost' => 'infrastructure/cluster/search/PhabricatorElasticSearchHost.php',
'PhabricatorElasticSearchQueryBuilder' => 'applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php',
'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php',
'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php',
'PhabricatorEmailContentSource' => 'applications/metamta/contentsource/PhabricatorEmailContentSource.php',
@ -3073,6 +3078,7 @@ phutil_register_library_map(array(
'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php',
'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php',
'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php',
'PhabricatorMySQLSearchHost' => 'infrastructure/cluster/search/PhabricatorMySQLSearchHost.php',
'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php',
'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php',
'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php',
@ -3762,7 +3768,6 @@ phutil_register_library_map(array(
'PhabricatorSearchApplicationStorageEnginePanel' => 'applications/search/applicationpanel/PhabricatorSearchApplicationStorageEnginePanel.php',
'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php',
'PhabricatorSearchCheckboxesField' => 'applications/search/field/PhabricatorSearchCheckboxesField.php',
'PhabricatorSearchConfigOptions' => 'applications/search/config/PhabricatorSearchConfigOptions.php',
'PhabricatorSearchConstraintException' => 'applications/search/exception/PhabricatorSearchConstraintException.php',
'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php',
'PhabricatorSearchCustomFieldProxyField' => 'applications/search/field/PhabricatorSearchCustomFieldProxyField.php',
@ -3785,6 +3790,7 @@ phutil_register_library_map(array(
'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php',
'PhabricatorSearchEngineTestCase' => 'applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php',
'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php',
'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php',
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
'PhabricatorSearchIndexVersion' => 'applications/search/storage/PhabricatorSearchIndexVersion.php',
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php',
@ -3804,6 +3810,7 @@ phutil_register_library_map(array(
'PhabricatorSearchSchemaSpec' => 'applications/search/storage/PhabricatorSearchSchemaSpec.php',
'PhabricatorSearchScopeSetting' => 'applications/settings/setting/PhabricatorSearchScopeSetting.php',
'PhabricatorSearchSelectField' => 'applications/search/field/PhabricatorSearchSelectField.php',
'PhabricatorSearchService' => 'infrastructure/cluster/search/PhabricatorSearchService.php',
'PhabricatorSearchStringListField' => 'applications/search/field/PhabricatorSearchStringListField.php',
'PhabricatorSearchSubscribersField' => 'applications/search/field/PhabricatorSearchSubscribersField.php',
'PhabricatorSearchTextField' => 'applications/search/field/PhabricatorSearchTextField.php',
@ -7303,6 +7310,9 @@ phutil_register_library_map(array(
'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler',
'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException',
'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException',
'PhabricatorClusterNoHostForRoleException' => 'Exception',
'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType',
'PhabricatorClusterServiceHealthRecord' => 'Phobject',
'PhabricatorClusterStrandedException' => 'PhabricatorClusterException',
'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
@ -7354,6 +7364,7 @@ phutil_register_library_map(array(
'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController',
'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController',
'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController',
'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController',
'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule',
'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema',
'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
@ -7624,7 +7635,6 @@ phutil_register_library_map(array(
'PhabricatorDashboardViewController' => 'PhabricatorDashboardProfileController',
'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec',
'PhabricatorDataNotAttachedException' => 'Exception',
'PhabricatorDatabaseHealthRecord' => 'Phobject',
'PhabricatorDatabaseRef' => 'Phobject',
'PhabricatorDatabaseRefParser' => 'Phobject',
'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
@ -7738,6 +7748,7 @@ phutil_register_library_map(array(
'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting',
'PhabricatorEditorSetting' => 'PhabricatorStringSetting',
'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
'PhabricatorElasticSearchHost' => 'PhabricatorSearchHost',
'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel',
'PhabricatorEmailContentSource' => 'PhabricatorContentSource',
@ -8208,6 +8219,7 @@ phutil_register_library_map(array(
'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine',
'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
'PhabricatorMySQLSearchHost' => 'PhabricatorSearchHost',
'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorNamedQuery' => array(
'PhabricatorSearchDAO',
@ -9074,7 +9086,6 @@ phutil_register_library_map(array(
'PhabricatorSearchApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel',
'PhabricatorSearchBaseController' => 'PhabricatorController',
'PhabricatorSearchCheckboxesField' => 'PhabricatorSearchField',
'PhabricatorSearchConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorSearchConstraintException' => 'Exception',
'PhabricatorSearchController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchCustomFieldProxyField' => 'PhabricatorSearchField',
@ -9097,6 +9108,7 @@ phutil_register_library_map(array(
'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule',
'PhabricatorSearchEngineTestCase' => 'PhabricatorTestCase',
'PhabricatorSearchField' => 'Phobject',
'PhabricatorSearchHost' => 'Phobject',
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchIndexVersion' => 'PhabricatorSearchDAO',
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',
@ -9116,6 +9128,7 @@ phutil_register_library_map(array(
'PhabricatorSearchSchemaSpec' => 'PhabricatorConfigSchemaSpec',
'PhabricatorSearchScopeSetting' => 'PhabricatorInternalSetting',
'PhabricatorSearchSelectField' => 'PhabricatorSearchField',
'PhabricatorSearchService' => 'Phobject',
'PhabricatorSearchStringListField' => 'PhabricatorSearchField',
'PhabricatorSearchSubscribersField' => 'PhabricatorSearchTokenizerField',
'PhabricatorSearchTextField' => 'PhabricatorSearchField',

View file

@ -69,6 +69,7 @@ final class PhabricatorConfigApplication extends PhabricatorApplication {
'databases/' => 'PhabricatorConfigClusterDatabasesController',
'notifications/' => 'PhabricatorConfigClusterNotificationsController',
'repositories/' => 'PhabricatorConfigClusterRepositoriesController',
'search/' => 'PhabricatorConfigClusterSearchController',
),
),
);

View file

@ -7,71 +7,74 @@ final class PhabricatorElasticSearchSetupCheck extends PhabricatorSetupCheck {
}
protected function executeChecks() {
if (!$this->shouldUseElasticSearchEngine()) {
return;
}
$services = PhabricatorSearchService::getAllServices();
$engine = new PhabricatorElasticFulltextStorageEngine();
$index_exists = null;
$index_sane = null;
try {
$index_exists = $engine->indexExists();
if ($index_exists) {
$index_sane = $engine->indexIsSane();
foreach ($services as $service) {
try {
$host = $service->getAnyHostForRole('read');
} catch (PhabricatorClusterNoHostForRoleException $e) {
// ignore the error
continue;
}
} catch (Exception $ex) {
$summary = pht('Elasticsearch is not reachable as configured.');
$message = pht(
'Elasticsearch is configured (with the %s setting) but Phabricator '.
'encountered an exception when trying to test the index.'.
"\n\n".
'%s',
phutil_tag('tt', array(), 'search.elastic.host'),
phutil_tag('pre', array(), $ex->getMessage()));
if ($host instanceof PhabricatorElasticSearchHost) {
$index_exists = null;
$index_sane = null;
try {
$engine = $host->getEngine();
$index_exists = $engine->indexExists();
if ($index_exists) {
$index_sane = $engine->indexIsSane();
}
} catch (Exception $ex) {
$summary = pht('Elasticsearch is not reachable as configured.');
$message = pht(
'Elasticsearch is configured (with the %s setting) but Phabricator'.
' encountered an exception when trying to test the index.'.
"\n\n".
'%s',
phutil_tag('tt', array(), 'cluster.search'),
phutil_tag('pre', array(), $ex->getMessage()));
$this->newIssue('elastic.misconfigured')
->setName(pht('Elasticsearch Misconfigured'))
->setSummary($summary)
->setMessage($message)
->addRelatedPhabricatorConfig('search.elastic.host');
return;
}
$this->newIssue('elastic.misconfigured')
->setName(pht('Elasticsearch Misconfigured'))
->setSummary($summary)
->setMessage($message)
->addRelatedPhabricatorConfig('cluster.search');
return;
}
if (!$index_exists) {
$summary = pht(
'You enabled Elasticsearch but the index does not exist.');
if (!$index_exists) {
$summary = pht(
'You enabled Elasticsearch but the index does not exist.');
$message = pht(
'You likely enabled search.elastic.host without creating the '.
'index. Run `./bin/search init` to correct the index.');
$message = pht(
'You likely enabled cluster.search without creating the '.
'index. Run `./bin/search init` to correct the index.');
$this
->newIssue('elastic.missing-index')
->setName(pht('Elasticsearch index Not Found'))
->setSummary($summary)
->setMessage($message)
->addRelatedPhabricatorConfig('search.elastic.host');
} else if (!$index_sane) {
$summary = pht(
'Elasticsearch index exists but needs correction.');
$this
->newIssue('elastic.missing-index')
->setName(pht('Elasticsearch index Not Found'))
->setSummary($summary)
->setMessage($message)
->addRelatedPhabricatorConfig('cluster.search');
} else if (!$index_sane) {
$summary = pht(
'Elasticsearch index exists but needs correction.');
$message = pht(
'Either the Phabricator schema for Elasticsearch has changed '.
'or Elasticsearch created the index automatically. Run '.
'`./bin/search init` to correct the index.');
$message = pht(
'Either the Phabricator schema for Elasticsearch has changed '.
'or Elasticsearch created the index automatically. Run '.
'`./bin/search init` to correct the index.');
$this
->newIssue('elastic.broken-index')
->setName(pht('Elasticsearch index Incorrect'))
->setSummary($summary)
->setMessage($message);
$this
->newIssue('elastic.broken-index')
->setName(pht('Elasticsearch index Incorrect'))
->setSummary($summary)
->setMessage($message);
}
}
}
}
protected function shouldUseElasticSearchEngine() {
$search_engine = PhabricatorFulltextStorageEngine::loadEngine();
return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine);
}
}

View file

@ -198,6 +198,10 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
'This option has been removed, you can use Dashboards to provide '.
'homepage customization. See T11533 for more details.');
$elastic_reason = pht(
'Elasticsearch is now configured with "%s".',
'cluster.search');
$ancient_config += array(
'phid.external-loaders' =>
pht(
@ -348,6 +352,10 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
'mysql.configuration-provider' => pht(
'Phabricator now has application-level management of partitioning '.
'and replicas.'),
'search.elastic.host' => $elastic_reason,
'search.elastic.namespace' => $elastic_reason,
);
return $ancient_config;

View file

@ -379,8 +379,13 @@ final class PhabricatorMySQLSetupCheck extends PhabricatorSetupCheck {
}
protected function shouldUseMySQLSearchEngine() {
$search_engine = PhabricatorFulltextStorageEngine::loadEngine();
return ($search_engine instanceof PhabricatorMySQLFulltextStorageEngine);
$services = PhabricatorSearchService::getAllServices();
foreach ($services as $service) {
if ($service instanceof PhabricatorMySQLSearchHost) {
return true;
}
}
return false;
}
}

View file

@ -0,0 +1,129 @@
<?php
final class PhabricatorConfigClusterSearchController
extends PhabricatorConfigController {
public function handleRequest(AphrontRequest $request) {
$nav = $this->buildSideNavView();
$nav->selectFilter('cluster/search/');
$title = pht('Cluster Search');
$doc_href = PhabricatorEnv::getDoclink('Cluster: Search');
$header = id(new PHUIHeaderView())
->setHeader($title)
->setProfileHeader(true)
->addActionLink(
id(new PHUIButtonView())
->setIcon('fa-book')
->setHref($doc_href)
->setTag('a')
->setText(pht('Documentation')));
$crumbs = $this
->buildApplicationCrumbs($nav)
->addTextCrumb($title)
->setBorder(true);
$search_status = $this->buildClusterSearchStatus();
$content = id(new PhabricatorConfigPageView())
->setHeader($header)
->setContent($search_status);
return $this->newPage()
->setTitle($title)
->setCrumbs($crumbs)
->setNavigation($nav)
->appendChild($content)
->addClass('white-background');
}
private function buildClusterSearchStatus() {
$viewer = $this->getViewer();
$services = PhabricatorSearchService::getAllServices();
Javelin::initBehavior('phabricator-tooltips');
$view = array();
foreach ($services as $service) {
$view[] = $this->renderStatusView($service);
}
return $view;
}
private function renderStatusView($service) {
$head = array_merge(
array(pht('Type')),
array_keys($service->getStatusViewColumns()),
array(pht('Status')));
$rows = array();
$status_map = PhabricatorSearchService::getConnectionStatusMap();
$stats = false;
$stats_view = false;
foreach ($service->getHosts() as $host) {
try {
$status = $host->getConnectionStatus();
$status = idx($status_map, $status, array());
} catch (Exception $ex) {
$status['icon'] = 'fa-times';
$status['label'] = pht('Connection Error');
$status['color'] = 'red';
$host->didHealthCheck(false);
}
if (!$stats_view) {
try {
$stats = $host->getEngine()->getIndexStats($host);
$stats_view = $this->renderIndexStats($stats);
} catch (Exception $e) {
$stats_view = false;
}
}
$type_icon = 'fa-search sky';
$type_tip = $host->getDisplayName();
$type_icon = id(new PHUIIconView())
->setIcon($type_icon);
$status_view = array(
id(new PHUIIconView())->setIcon($status['icon'].' '.$status['color']),
' ',
$status['label'],
);
$row = array(array($type_icon, ' ', $type_tip));
$row = array_merge($row, array_values(
$host->getStatusViewColumns()));
$row[] = $status_view;
$rows[] = $row;
}
$table = id(new AphrontTableView($rows))
->setNoDataString(pht('No search servers are configured.'))
->setHeaders($head);
$view = id(new PHUIObjectBoxView())
->setHeaderText($service->getDisplayName())
->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
->setTable($table);
if ($stats_view) {
$view->addPropertyList($stats_view);
}
return $view;
}
private function renderIndexStats($stats) {
$view = id(new PHUIPropertyListView());
if ($stats !== false) {
foreach ($stats as $label => $val) {
$view->addProperty($label, $val);
}
}
return $view;
}
}

View file

@ -42,8 +42,11 @@ abstract class PhabricatorConfigController extends PhabricatorController {
pht('Notification Servers'), null, 'fa-bell-o');
$nav->addFilter('cluster/repositories/',
pht('Repository Servers'), null, 'fa-code');
$nav->addFilter('cluster/search/',
pht('Search Servers'), null, 'fa-search');
$nav->addLabel(pht('Modules'));
$modules = PhabricatorConfigModule::getAllModules();
foreach ($modules as $key => $module) {
$nav->addFilter('module/'.$key.'/',

View file

@ -38,6 +38,17 @@ EOTEXT
$intro_href = PhabricatorEnv::getDoclink('Clustering Introduction');
$intro_name = pht('Clustering Introduction');
$search_type = 'custom:PhabricatorClusterSearchConfigOptionType';
$search_help = $this->deformat(pht(<<<EOTEXT
Define one or more fulltext storage services. Here you can configure which
hosts will handle fulltext search queries and indexing. For help with
configuring fulltext search clusters, see **[[ %s | %s ]]** in the
documentation.
EOTEXT
,
PhabricatorEnv::getDoclink('Cluster: Search'),
pht('Cluster: Search')));
return array(
$this->newOption('cluster.addresses', 'list<string>', array())
->setLocked(true)
@ -114,6 +125,21 @@ EOTEXT
->setSummary(
pht('Configure database read replicas.'))
->setDescription($databases_help),
$this->newOption('cluster.search', $search_type, array())
->setLocked(true)
->setSummary(
pht('Configure full-text search services.'))
->setDescription($search_help)
->setDefault(
array(
array(
'type' => 'mysql',
'roles' => array(
'read' => true,
'write' => true,
),
),
)),
);
}

View file

@ -513,14 +513,14 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery {
->setEngineClassName('PhabricatorSearchApplicationSearchEngine')
->setParameter('query', $this->fullTextSearch);
// NOTE: Setting this to something larger than 2^53 will raise errors in
// NOTE: Setting this to something larger than 10,000 will raise errors in
// ElasticSearch, and billions of results won't fit in memory anyway.
$fulltext_query->setParameter('limit', 100000);
$fulltext_query->setParameter('limit', 10000);
$fulltext_query->setParameter('types',
array(ManiphestTaskPHIDType::TYPECONST));
$engine = PhabricatorFulltextStorageEngine::loadEngine();
$fulltext_results = $engine->executeSearch($fulltext_query);
$fulltext_results = PhabricatorSearchService::executeSearch(
$fulltext_query);
if (empty($fulltext_results)) {
$fulltext_results = array(null);

View file

@ -10,7 +10,15 @@ final class PhabricatorProjectFulltextEngine
$project = $object;
$project->updateDatasourceTokens();
$document->setDocumentTitle($project->getName());
$document->setDocumentTitle($project->getDisplayName());
$document->addField(PhabricatorSearchDocumentFieldType::FIELD_KEYWORDS,
$project->getPrimarySlug());
try {
$slugs = $project->getSlugs();
foreach ($slugs as $slug) {}
} catch (PhabricatorDataNotAttachedException $e) {
// ignore
}
$document->addRelationship(
$project->isArchived()

View file

@ -1,35 +0,0 @@
<?php
final class PhabricatorSearchConfigOptions
extends PhabricatorApplicationConfigOptions {
public function getName() {
return pht('Search');
}
public function getDescription() {
return pht('Options relating to Search.');
}
public function getIcon() {
return 'fa-search';
}
public function getGroup() {
return 'apps';
}
public function getOptions() {
return array(
$this->newOption('search.elastic.host', 'string', null)
->setLocked(true)
->setDescription(pht('Elastic Search host.'))
->addExample('http://elastic.example.com:9200/', pht('Valid Setting')),
$this->newOption('search.elastic.namespace', 'string', 'phabricator')
->setLocked(true)
->setDescription(pht('Elastic Search index.'))
->addExample('phabricator2', pht('Valid Setting')),
);
}
}

View file

@ -5,5 +5,6 @@ final class PhabricatorSearchDocumentFieldType extends Phobject {
const FIELD_TITLE = 'titl';
const FIELD_BODY = 'body';
const FIELD_COMMENT = 'cmnt';
const FIELD_KEYWORDS = 'kwrd';
}

View file

@ -3,8 +3,8 @@
final class PhabricatorSearchEngineTestCase extends PhabricatorTestCase {
public function testLoadAllEngines() {
PhabricatorFulltextStorageEngine::loadAllEngines();
$this->assertTrue(true);
$services = PhabricatorSearchService::getAllServices();
$this->assertTrue(!empty($services));
}
}

View file

@ -1,37 +1,52 @@
<?php
final class PhabricatorElasticFulltextStorageEngine
class PhabricatorElasticFulltextStorageEngine
extends PhabricatorFulltextStorageEngine {
private $uri;
private $index;
private $timeout;
private $version;
public function __construct() {
$this->uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
$this->index = PhabricatorEnv::getEnvConfig('search.elastic.namespace');
public function setService(PhabricatorSearchService $service) {
$this->service = $service;
$config = $service->getConfig();
$index = idx($config, 'path', '/phabricator');
$this->index = str_replace('/', '', $index);
$this->timeout = idx($config, 'timeout', 15);
$this->version = (int)idx($config, 'version', 5);
return $this;
}
public function getEngineIdentifier() {
return 'elasticsearch';
}
public function getEnginePriority() {
return 10;
public function getTimestampField() {
return $this->version < 2 ?
'_timestamp' : 'lastModified';
}
public function isEnabled() {
return (bool)$this->uri;
public function getTextFieldType() {
return $this->version >= 5
? 'text' : 'string';
}
public function setURI($uri) {
$this->uri = $uri;
return $this;
public function getHostType() {
return new PhabricatorElasticSearchHost($this);
}
public function setIndex($index) {
$this->index = $index;
return $this;
/**
* @return PhabricatorElasticSearchHost
*/
public function getHostForRead() {
return $this->getService()->getAnyHostForRole('read');
}
/**
* @return PhabricatorElasticSearchHost
*/
public function getHostForWrite() {
return $this->getService()->getAnyHostForRole('write');
}
public function setTimeout($timeout) {
@ -39,21 +54,21 @@ final class PhabricatorElasticFulltextStorageEngine
return $this;
}
public function getURI() {
return $this->uri;
}
public function getIndex() {
return $this->index;
}
public function getTimeout() {
return $this->timeout;
}
public function getTypeConstants($class) {
$relationship_class = new ReflectionClass($class);
$typeconstants = $relationship_class->getConstants();
return array_unique(array_values($typeconstants));
}
public function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $doc) {
$host = $this->getHostForWrite();
$type = $doc->getDocumentType();
$phid = $doc->getPHID();
$handle = id(new PhabricatorHandleQuery())
@ -61,36 +76,47 @@ final class PhabricatorElasticFulltextStorageEngine
->withPHIDs(array($phid))
->executeOne();
$timestamp_key = $this->getTimestampField();
// URL is not used internally but it can be useful externally.
$spec = array(
'title' => $doc->getDocumentTitle(),
'url' => PhabricatorEnv::getProductionURI($handle->getURI()),
'dateCreated' => $doc->getDocumentCreated(),
'_timestamp' => $doc->getDocumentModified(),
'field' => array(),
'relationship' => array(),
$timestamp_key => $doc->getDocumentModified(),
);
foreach ($doc->getFieldData() as $field) {
$spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field);
list($field_name, $corpus, $aux) = $field;
if (!isset($spec[$field_name])) {
$spec[$field_name] = array($corpus);
} else {
$spec[$field_name][] = $corpus;
}
if ($aux != null) {
$spec[$field_name][] = $aux;
}
}
foreach ($doc->getRelationshipData() as $relationship) {
list($rtype, $to_phid, $to_type, $time) = $relationship;
$spec['relationship'][$rtype][] = array(
'phid' => $to_phid,
'phidType' => $to_type,
'when' => (int)$time,
);
foreach ($doc->getRelationshipData() as $field) {
list($field_name, $related_phid, $rtype, $time) = $field;
if (!isset($spec[$field_name])) {
$spec[$field_name] = array($related_phid);
} else {
$spec[$field_name][] = $related_phid;
}
if ($time) {
$spec[$field_name.'_ts'] = $time;
}
}
$this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT');
$this->executeRequest($host, "/{$type}/{$phid}/", $spec, 'PUT');
}
public function reconstructDocument($phid) {
$type = phid_get_type($phid);
$response = $this->executeRequest("/{$type}/{$phid}", array());
$host = $this->getHostForRead();
$response = $this->executeRequest($host, "/{$type}/{$phid}", array());
if (empty($response['exists'])) {
return null;
@ -103,10 +129,11 @@ final class PhabricatorElasticFulltextStorageEngine
$doc->setDocumentType($response['_type']);
$doc->setDocumentTitle($hit['title']);
$doc->setDocumentCreated($hit['dateCreated']);
$doc->setDocumentModified($hit['_timestamp']);
$doc->setDocumentModified($hit[$this->getTimestampField()]);
foreach ($hit['field'] as $fdef) {
$doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']);
$field_type = $fdef['type'];
$doc->addField($field_type, $hit[$field_type], $fdef['aux']);
}
foreach ($hit['relationship'] as $rtype => $rships) {
@ -123,35 +150,51 @@ final class PhabricatorElasticFulltextStorageEngine
}
private function buildSpec(PhabricatorSavedQuery $query) {
$spec = array();
$filter = array();
$title_spec = array();
$q = new PhabricatorElasticSearchQueryBuilder('bool');
$query_string = $query->getParameter('query');
if (strlen($query_string)) {
$fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType');
if (strlen($query->getParameter('query'))) {
$spec[] = array(
// Build a simple_query_string query over all fields that must match all
// of the words in the search string.
$q->addMustClause(array(
'simple_query_string' => array(
'query' => $query->getParameter('query'),
'fields' => array('field.corpus'),
'query' => $query_string,
'fields' => array(
'_all',
),
'default_operator' => 'OR',
),
);
));
$title_spec = array(
// This second query clause is "SHOULD' so it only affects ranking of
// documents which already matched the Must clause. This amplifies the
// score of documents which have an exact match on title, body
// or comments.
$q->addShouldClause(array(
'simple_query_string' => array(
'query' => $query->getParameter('query'),
'fields' => array('title'),
'query' => $query_string,
'fields' => array(
PhabricatorSearchDocumentFieldType::FIELD_TITLE.'^4',
PhabricatorSearchDocumentFieldType::FIELD_BODY.'^3',
PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'^1.2',
),
'analyzer' => 'english_exact',
'default_operator' => 'and',
),
);
));
}
$exclude = $query->getParameter('exclude');
if ($exclude) {
$filter[] = array(
$q->addFilterClause(array(
'not' => array(
'ids' => array(
'values' => array($exclude),
),
),
);
));
}
$relationship_map = array(
@ -176,75 +219,59 @@ final class PhabricatorElasticFulltextStorageEngine
$include_closed = !empty($statuses[$rel_closed]);
if ($include_open && !$include_closed) {
$relationship_map[$rel_open] = true;
$q->addExistsClause($rel_open);
} else if (!$include_open && $include_closed) {
$relationship_map[$rel_closed] = true;
$q->addExistsClause($rel_closed);
}
if ($query->getParameter('withUnowned')) {
$relationship_map[$rel_unowned] = true;
$q->addExistsClause($rel_unowned);
}
$rel_owner = PhabricatorSearchRelationship::RELATIONSHIP_OWNER;
if ($query->getParameter('withAnyOwner')) {
$relationship_map[$rel_owner] = true;
$q->addExistsClause($rel_owner);
} else {
$owner_phids = $query->getParameter('ownerPHIDs', array());
$relationship_map[$rel_owner] = $owner_phids;
}
foreach ($relationship_map as $field => $param) {
if (is_array($param) && $param) {
$should = array();
foreach ($param as $val) {
$should[] = array(
'match' => array(
"relationship.{$field}.phid" => array(
'query' => $val,
'type' => 'phrase',
),
),
);
}
// We couldn't solve it by minimum_number_should_match because it can
// match multiple owners without matching author.
$spec[] = array('bool' => array('should' => $should));
} else if ($param) {
$filter[] = array(
'exists' => array(
'field' => "relationship.{$field}.phid",
),
);
if (count($owner_phids)) {
$q->addTermsClause($rel_owner, $owner_phids);
}
}
if ($spec) {
$spec = array('query' => array('bool' => array('must' => $spec)));
if ($title_spec) {
$spec['query']['bool']['should'] = $title_spec;
foreach ($relationship_map as $field => $phids) {
if (is_array($phids) && !empty($phids)) {
$q->addTermsClause($field, $phids);
}
}
if ($filter) {
$filter = array('filter' => array('and' => $filter));
if (!$spec) {
$spec = array('query' => array('match_all' => new stdClass()));
}
$spec = array(
'query' => array(
'filtered' => $spec + $filter,
),
);
if (!$q->getClauseCount('must')) {
$q->addMustClause(array('match_all' => array('boost' => 1 )));
}
$spec = array(
'_source' => false,
'query' => array(
'bool' => $q->toArray(),
),
);
if (!$query->getParameter('query')) {
$spec['sort'] = array(
array('dateCreated' => 'desc'),
);
}
$spec['from'] = (int)$query->getParameter('offset', 0);
$spec['size'] = (int)$query->getParameter('limit', 25);
$offset = (int)$query->getParameter('offset', 0);
$limit = (int)$query->getParameter('limit', 101);
if ($offset + $limit > 10000) {
throw new Exception(pht(
'Query offset is too large. offset+limit=%s (max=%s)',
$offset + $limit,
10000));
}
$spec['from'] = $offset;
$spec['size'] = $limit;
return $spec;
}
@ -261,30 +288,36 @@ final class PhabricatorElasticFulltextStorageEngine
// some bigger index). Use '/$types/_search' instead.
$uri = '/'.implode(',', $types).'/_search';
try {
$response = $this->executeRequest($uri, $this->buildSpec($query));
} catch (HTTPFutureHTTPResponseStatus $ex) {
// elasticsearch probably uses Lucene query syntax:
// http://lucene.apache.org/core/3_6_1/queryparsersyntax.html
// Try literal search if operator search fails.
if (!strlen($query->getParameter('query'))) {
throw $ex;
}
$query = clone $query;
$query->setParameter(
'query',
addcslashes(
$query->getParameter('query'), '+-&|!(){}[]^"~*?:\\'));
$response = $this->executeRequest($uri, $this->buildSpec($query));
}
$spec = $this->buildSpec($query);
$exceptions = array();
$phids = ipull($response['hits']['hits'], '_id');
return $phids;
foreach ($this->service->getAllHostsForRole('read') as $host) {
try {
$response = $this->executeRequest($host, $uri, $spec);
$phids = ipull($response['hits']['hits'], '_id');
return $phids;
} catch (Exception $e) {
$exceptions[] = $e;
}
}
throw new PhutilAggregateException('All search hosts failed:', $exceptions);
}
public function indexExists() {
public function indexExists(PhabricatorElasticSearchHost $host = null) {
if (!$host) {
$host = $this->getHostForRead();
}
try {
return (bool)$this->executeRequest('/_status/', array());
if ($this->version >= 5) {
$uri = '/_stats/';
$res = $this->executeRequest($host, $uri, array());
return isset($res['indices']['phabricator']);
} else if ($this->version >= 2) {
$uri = '';
} else {
$uri = '/_status/';
}
return (bool)$this->executeRequest($host, $uri, array());
} catch (HTTPFutureHTTPResponseStatus $e) {
if ($e->getStatusCode() == 404) {
return false;
@ -299,53 +332,85 @@ final class PhabricatorElasticFulltextStorageEngine
'index' => array(
'auto_expand_replicas' => '0-2',
'analysis' => array(
'filter' => array(
'trigrams_filter' => array(
'min_gram' => 3,
'type' => 'ngram',
'max_gram' => 3,
),
),
'analyzer' => array(
'custom_trigrams' => array(
'type' => 'custom',
'filter' => array(
'lowercase',
'kstem',
'trigrams_filter',
),
'english_exact' => array(
'tokenizer' => 'standard',
'filter' => array('lowercase'),
),
),
),
),
);
$types = array_keys(
$fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType');
$relationships = $this->getTypeConstants('PhabricatorSearchRelationship');
$doc_types = array_keys(
PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes());
foreach ($types as $type) {
// Use the custom trigram analyzer for the corpus of text
$data['mappings'][$type]['properties']['field']['properties']['corpus'] =
array('type' => 'string', 'analyzer' => 'custom_trigrams');
$text_type = $this->getTextFieldType();
foreach ($doc_types as $type) {
$properties = array();
foreach ($fields as $field) {
// Use the custom analyzer for the corpus of text
$properties[$field] = array(
'type' => $text_type,
'analyzer' => 'english_exact',
'search_analyzer' => 'english',
'search_quote_analyzer' => 'english_exact',
);
}
if ($this->version < 5) {
foreach ($relationships as $rel) {
$properties[$rel] = array(
'type' => 'string',
'index' => 'not_analyzed',
'include_in_all' => false,
);
$properties[$rel.'_ts'] = array(
'type' => 'date',
'include_in_all' => false,
);
}
} else {
foreach ($relationships as $rel) {
$properties[$rel] = array(
'type' => 'keyword',
'include_in_all' => false,
'doc_values' => false,
);
$properties[$rel.'_ts'] = array(
'type' => 'date',
'include_in_all' => false,
);
}
}
// Ensure we have dateCreated since the default query requires it
$data['mappings'][$type]['properties']['dateCreated']['type'] = 'string';
}
$properties['dateCreated']['type'] = 'date';
$properties['lastModified']['type'] = 'date';
$data['mappings'][$type]['properties'] = $properties;
}
return $data;
}
public function indexIsSane() {
if (!$this->indexExists()) {
public function indexIsSane(PhabricatorElasticSearchHost $host = null) {
if (!$host) {
$host = $this->getHostForRead();
}
if (!$this->indexExists($host)) {
return false;
}
$cur_mapping = $this->executeRequest('/_mapping/', array());
$cur_settings = $this->executeRequest('/_settings/', array());
$cur_mapping = $this->executeRequest($host, '/_mapping/', array());
$cur_settings = $this->executeRequest($host, '/_settings/', array());
$actual = array_merge($cur_settings[$this->index],
$cur_mapping[$this->index]);
return $this->check($actual, $this->getIndexConfiguration());
$res = $this->check($actual, $this->getIndexConfiguration());
return $res;
}
/**
@ -355,7 +420,7 @@ final class PhabricatorElasticFulltextStorageEngine
* @param $required array
* @return bool
*/
private function check($actual, $required) {
private function check($actual, $required, $path = '') {
foreach ($required as $key => $value) {
if (!array_key_exists($key, $actual)) {
if ($key === '_all') {
@ -369,7 +434,7 @@ final class PhabricatorElasticFulltextStorageEngine
if (!is_array($actual[$key])) {
return false;
}
if (!$this->check($actual[$key], $value)) {
if (!$this->check($actual[$key], $value, $path.'.'.$key)) {
return false;
}
continue;
@ -403,19 +468,44 @@ final class PhabricatorElasticFulltextStorageEngine
}
public function initIndex() {
$host = $this->getHostForWrite();
if ($this->indexExists()) {
$this->executeRequest('/', array(), 'DELETE');
$this->executeRequest($host, '/', array(), 'DELETE');
}
$data = $this->getIndexConfiguration();
$this->executeRequest('/', $data, 'PUT');
$this->executeRequest($host, '/', $data, 'PUT');
}
private function executeRequest($path, array $data, $method = 'GET') {
$uri = new PhutilURI($this->uri);
$uri->setPath($this->index);
$uri->appendPath($path);
$data = json_encode($data);
public function getIndexStats(PhabricatorElasticSearchHost $host = null) {
if ($this->version < 2) {
return false;
}
if (!$host) {
$host = $this->getHostForRead();
}
$uri = '/_stats/';
$host = $this->getHostForRead();
$res = $this->executeRequest($host, $uri, array());
$stats = $res['indices'][$this->index];
return array(
pht('Queries') =>
idxv($stats, array('primaries', 'search', 'query_total')),
pht('Documents') =>
idxv($stats, array('total', 'docs', 'count')),
pht('Deleted') =>
idxv($stats, array('total', 'docs', 'deleted')),
pht('Storage Used') =>
phutil_format_bytes(idxv($stats,
array('total', 'store', 'size_in_bytes'))),
);
}
private function executeRequest(PhabricatorElasticSearchHost $host, $path,
array $data, $method = 'GET') {
$uri = $host->getURI($path);
$data = json_encode($data);
$future = new HTTPSFuture($uri, $data);
if ($method != 'GET') {
$future->setMethod($method);
@ -423,19 +513,30 @@ final class PhabricatorElasticFulltextStorageEngine
if ($this->getTimeout()) {
$future->setTimeout($this->getTimeout());
}
list($body) = $future->resolvex();
try {
list($body) = $future->resolvex();
} catch (HTTPFutureResponseStatus $ex) {
if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) {
$host->didHealthCheck(false);
}
throw $ex;
}
if ($method != 'GET') {
return null;
}
try {
return phutil_json_decode($body);
$data = phutil_json_decode($body);
$host->didHealthCheck(true);
return $data;
} catch (PhutilJSONParserException $ex) {
$host->didHealthCheck(false);
throw new PhutilProxyException(
pht('ElasticSearch server returned invalid JSON!'),
$ex);
}
}
}

View file

@ -0,0 +1,78 @@
<?php
class PhabricatorElasticSearchQueryBuilder {
protected $name;
protected $clauses = array();
public function getClauses($termkey = null) {
$clauses = $this->clauses;
if ($termkey == null) {
return $clauses;
}
if (isset($clauses[$termkey])) {
return $clauses[$termkey];
}
return array();
}
public function getClauseCount($clausekey) {
if (isset($this->clauses[$clausekey])) {
return count($this->clauses[$clausekey]);
} else {
return 0;
}
}
public function addExistsClause($field) {
return $this->addClause('filter', array(
'exists' => array(
'field' => $field,
),
));
}
public function addTermsClause($field, $values) {
return $this->addClause('filter', array(
'terms' => array(
$field => array_values($values),
),
));
}
public function addMustClause($clause) {
return $this->addClause('must', $clause);
}
public function addFilterClause($clause) {
return $this->addClause('filter', $clause);
}
public function addShouldClause($clause) {
return $this->addClause('should', $clause);
}
public function addMustNotClause($clause) {
return $this->addClause('must_not', $clause);
}
public function addClause($clause, $terms) {
$this->clauses[$clause][] = $terms;
return $this;
}
public function toArray() {
$clauses = $this->getClauses();
return $clauses;
$cleaned = array();
foreach ($clauses as $clause => $subclauses) {
if (is_array($subclauses) && count($subclauses) == 1) {
$cleaned[$clause] = array_shift($subclauses);
} else {
$cleaned[$clause] = $subclauses;
}
}
return $cleaned;
}
}

View file

@ -7,6 +7,31 @@
*/
abstract class PhabricatorFulltextStorageEngine extends Phobject {
protected $service;
public function getHosts() {
return $this->service->getHosts();
}
public function setService(PhabricatorSearchService $service) {
$this->service = $service;
return $this;
}
/**
* @return PhabricatorSearchService
*/
public function getService() {
return $this->service;
}
/**
* Implementations must return a prototype host instance which is cloned
* by the PhabricatorSearchService infrastructure to configure each engine.
* @return PhabricatorSearchHost
*/
abstract public function getHostType();
/* -( Engine Metadata )---------------------------------------------------- */
/**
@ -17,37 +42,6 @@ abstract class PhabricatorFulltextStorageEngine extends Phobject {
*/
abstract public function getEngineIdentifier();
/**
* Prioritize this engine relative to other engines.
*
* Engines with a smaller priority number get an opportunity to write files
* first. Generally, lower-latency filestores should have lower priority
* numbers, and higher-latency filestores should have higher priority
* numbers. Setting priority to approximately the number of milliseconds of
* read latency will generally produce reasonable results.
*
* In conjunction with filesize limits, the goal is to store small files like
* profile images, thumbnails, and text snippets in lower-latency engines,
* and store large files in higher-capacity engines.
*
* @return float Engine priority.
* @task meta
*/
abstract public function getEnginePriority();
/**
* Return `true` if the engine is currently writable.
*
* Engines that are disabled or missing configuration should return `false`
* to prevent new writes. If writes were made with this engine in the past,
* the application may still try to perform reads.
*
* @return bool True if this engine can support new writes.
* @task meta
*/
abstract public function isEnabled();
/* -( Managing Documents )------------------------------------------------- */
/**
@ -83,6 +77,13 @@ abstract class PhabricatorFulltextStorageEngine extends Phobject {
*/
abstract public function indexExists();
/**
* Implementations should override this method to return a dictionary of
* stats which are suitable for display in the admin UI.
*/
abstract public function getIndexStats();
/**
* Is the index in a usable state?
*
@ -100,39 +101,4 @@ abstract class PhabricatorFulltextStorageEngine extends Phobject {
public function initIndex() {}
/* -( Loading Storage Engines )-------------------------------------------- */
/**
* @task load
*/
public static function loadAllEngines() {
return id(new PhutilClassMapQuery())
->setAncestorClass(__CLASS__)
->setUniqueMethod('getEngineIdentifier')
->setSortMethod('getEnginePriority')
->execute();
}
/**
* @task load
*/
public static function loadActiveEngines() {
$engines = self::loadAllEngines();
$active = array();
foreach ($engines as $key => $engine) {
if (!$engine->isEnabled()) {
continue;
}
$active[$key] = $engine;
}
return $active;
}
public static function loadEngine() {
return head(self::loadActiveEngines());
}
}

View file

@ -7,12 +7,8 @@ final class PhabricatorMySQLFulltextStorageEngine
return 'mysql';
}
public function getEnginePriority() {
return 100;
}
public function isEnabled() {
return true;
public function getHostType() {
return new PhabricatorMySQLSearchHost($this);
}
public function reindexAbstractDocument(
@ -415,4 +411,9 @@ final class PhabricatorMySQLFulltextStorageEngine
public function indexExists() {
return true;
}
public function getIndexStats() {
return false;
}
}

View file

@ -40,8 +40,7 @@ abstract class PhabricatorFulltextEngine
$extension->indexFulltextObject($object, $document);
}
$storage_engine = PhabricatorFulltextStorageEngine::loadEngine();
$storage_engine->reindexAbstractDocument($document);
PhabricatorSearchService::reindexAbstractDocument($document);
}
protected function newAbstractDocument($object) {

View file

@ -13,27 +13,41 @@ final class PhabricatorSearchManagementInitWorkflow
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
$engine = PhabricatorFulltextStorageEngine::loadEngine();
$work_done = false;
if (!$engine->indexExists()) {
$console->writeOut(
'%s',
pht('Index does not exist, creating...'));
$engine->initIndex();
foreach (PhabricatorSearchService::getAllServices() as $service) {
$console->writeOut(
"%s\n",
pht('done.'));
$work_done = true;
} else if (!$engine->indexIsSane()) {
$console->writeOut(
'%s',
pht('Index exists but is incorrect, fixing...'));
$engine->initIndex();
$console->writeOut(
"%s\n",
pht('done.'));
$work_done = true;
pht('Initializing search service "%s"', $service->getDisplayName()));
try {
$host = $service->getAnyHostForRole('write');
} catch (PhabricatorClusterNoHostForRoleException $e) {
// If there are no writable hosts for a given cluster, skip it
$console->writeOut("%s\n", $e->getMessage());
continue;
}
$engine = $host->getEngine();
if (!$engine->indexExists()) {
$console->writeOut(
'%s',
pht('Index does not exist, creating...'));
$engine->initIndex();
$console->writeOut(
"%s\n",
pht('done.'));
$work_done = true;
} else if (!$engine->indexIsSane()) {
$console->writeOut(
'%s',
pht('Index exists but is incorrect, fixing...'));
$engine->initIndex();
$console->writeOut(
"%s\n",
pht('done.'));
$work_done = true;
}
}
if ($work_done) {

View file

@ -73,10 +73,7 @@ final class PhabricatorSearchDocumentQuery
$query = id(clone($this->savedQuery))
->setParameter('offset', $this->getOffset())
->setParameter('limit', $this->getRawResultLimit());
$engine = PhabricatorFulltextStorageEngine::loadEngine();
return $engine->executeSearch($query);
return PhabricatorSearchService::executeSearch($query);
}
public function getQueryApplicationClass() {

View file

@ -0,0 +1,76 @@
@title Cluster: Search
@group cluster
Overview
========
You can configure phabricator to connect to one or more fulltext search clusters
running either Elasticsearch or MySQL. By default and without further
configuration, Phabricator will use MySQL for fulltext search. This will be
adequate for the vast majority of users. Installs with a very large number of
objects or specialized search needs can consider enabling Elasticsearch for
better scalability and potentially better search results.
Configuring Search Services
===========================
To configure an Elasticsearch service, use the `cluster.search` configuration
option. A typical Elasticsearch configuration will probably look similar to
the following example:
```lang=json
{
"cluster.search": [
{
"type": "elasticsearch",
"hosts": [
{
"host": "127.0.0.1",
"roles": { "write": true, "read": true }
}
],
"port": 9200,
"protocol": "http",
"path": "/phabricator",
"version": 5
},
],
}
```
Supported Options
-----------------
| Key | Type |Comments|
|`type` | String |Engine type. Currently, 'elasticsearch' or 'mysql'|
|`protocol`| String |Either 'http' or 'https'|
|`port`| Int |The TCP port that Elasticsearch is bound to|
|`path`| String |The path portion of the url for phabricator's index.|
|`version`| Int |The version of Elasticsearch server. Supports either 2 or 5.|
|`hosts`| List |A list of one or more Elasticsearch host names / addresses.|
Host Configuration
------------------
Each search service must have one or more hosts associated with it. Each host
entry consists of a `host` key, a dictionary of roles and can optionally
override any of the options that are valid at the service level (see above).
Currently supported roles are `read` and `write`. These can be individually
enabled or disabled on a per-host basis. A typical setup might include two
elasticsearch clusters in two separate datacenters. You can configure one
cluster for reads and both for writes. When one cluster is down for maintenance
you can simply swap the read role over to the backup cluster and then proceed
with maintenance without any service interruption.
Monitoring Search Services
==========================
You can monitor fulltext search in {nav Config > Search Servers}. This interface
shows you a quick overview of services and their health.
The table on this page shows some basic stats for each configured service,
followed by the configuration and current status of each host.
NOTE: This page runs its diagnostics //from the web server that is serving the
request//. If you are recovering from a disaster, the view this page shows
may be partial or misleading, and two requests served by different servers may
see different views of the cluster.

View file

@ -1,20 +1,19 @@
<?php
final class PhabricatorDatabaseHealthRecord
class PhabricatorClusterServiceHealthRecord
extends Phobject {
private $ref;
private $cacheKey;
private $shouldCheck;
private $isHealthy;
private $upEventCount;
private $downEventCount;
public function __construct(PhabricatorDatabaseRef $ref) {
$this->ref = $ref;
public function __construct($cache_key) {
$this->cacheKey = $cache_key;
$this->readState();
}
/**
* Is the database currently healthy?
*/
@ -153,18 +152,13 @@ final class PhabricatorDatabaseHealthRecord
}
}
private function getHealthRecordCacheKey() {
$ref = $this->ref;
$host = $ref->getHost();
$port = $ref->getPort();
return "cluster.db.health({$host}, {$port})";
public function getCacheKey() {
return $this->cacheKey;
}
private function readHealthRecord() {
$cache = PhabricatorCaches::getSetupCache();
$cache_key = $this->getHealthRecordCacheKey();
$cache_key = $this->getCacheKey();
$health_record = $cache->getKey($cache_key);
if (!is_array($health_record)) {
@ -180,7 +174,7 @@ final class PhabricatorDatabaseHealthRecord
private function writeHealthRecord(array $record) {
$cache = PhabricatorCaches::getSetupCache();
$cache_key = $this->getHealthRecordCacheKey();
$cache_key = $this->getCacheKey();
$cache->setKey($cache_key, $record);
}

View file

@ -14,6 +14,7 @@ final class PhabricatorDatabaseRef
const REPLICATION_SLOW = 'replica-slow';
const REPLICATION_NOT_REPLICATING = 'not-replicating';
const KEY_HEALTH = 'cluster.db.health';
const KEY_REFS = 'cluster.db.refs';
const KEY_INDIVIDUAL = 'cluster.db.individual';
@ -489,9 +490,18 @@ final class PhabricatorDatabaseRef
return $this;
}
private function getHealthRecordCacheKey() {
$host = $this->getHost();
$port = $this->getPort();
$key = self::KEY_HEALTH;
return "{$key}({$host}, {$port})";
}
public function getHealthRecord() {
if (!$this->healthRecord) {
$this->healthRecord = new PhabricatorDatabaseHealthRecord($this);
$this->healthRecord = new PhabricatorClusterServiceHealthRecord(
$this->getHealthRecordCacheKey());
}
return $this->healthRecord;
}

View file

@ -0,0 +1,79 @@
<?php
final class PhabricatorClusterSearchConfigOptionType
extends PhabricatorConfigJSONOptionType {
public function validateOption(PhabricatorConfigOption $option, $value) {
if (!is_array($value)) {
throw new Exception(
pht(
'Search cluster configuration is not valid: value must be a '.
'list of search hosts.'));
}
$engines = PhabricatorSearchService::loadAllFulltextStorageEngines();
foreach ($value as $index => $spec) {
if (!is_array($spec)) {
throw new Exception(
pht(
'Search cluster configuration is not valid: each entry in the '.
'list must be a dictionary describing a search service, but '.
'the value with index "%s" is not a dictionary.',
$index));
}
try {
PhutilTypeSpec::checkMap(
$spec,
array(
'type' => 'string',
'hosts' => 'optional list<map<string, wild>>',
'roles' => 'optional map<string, wild>',
'port' => 'optional int',
'protocol' => 'optional string',
'path' => 'optional string',
'version' => 'optional int',
));
} catch (Exception $ex) {
throw new Exception(
pht(
'Search engine configuration has an invalid service '.
'specification (at index "%s"): %s.',
$index,
$ex->getMessage()));
}
if (!array_key_exists($spec['type'], $engines)) {
throw new Exception(
pht('Invalid search engine type: %s. Valid types include: %s',
$spec['type'],
implode(', ', array_keys($engines))));
}
if (isset($spec['hosts'])) {
foreach ($spec['hosts'] as $hostindex => $host) {
try {
PhutilTypeSpec::checkMap(
$host,
array(
'host' => 'string',
'roles' => 'optional map<string, wild>',
'port' => 'optional int',
'protocol' => 'optional string',
'path' => 'optional string',
'version' => 'optional int',
));
} catch (Exception $ex) {
throw new Exception(
pht(
'Search cluster configuration has an invalid host '.
'specification (at index "%s"): %s.',
$hostindex,
$ex->getMessage()));
}
}
}
}
}
}

View file

@ -0,0 +1,10 @@
<?php
final class PhabricatorClusterNoHostForRoleException
extends Exception {
public function __construct($role) {
parent::__construct(pht('Search cluster has no hosts for role "%s".',
$role));
}
}

View file

@ -0,0 +1,82 @@
<?php
final class PhabricatorElasticSearchHost
extends PhabricatorSearchHost {
private $version = 5;
private $path = 'phabricator/';
private $protocol = 'http';
const KEY_REFS = 'search.elastic.refs';
public function setConfig($config) {
$this->setRoles(idx($config, 'roles', $this->getRoles()))
->setHost(idx($config, 'host', $this->host))
->setPort(idx($config, 'port', $this->port))
->setProtocol(idx($config, 'protocol', $this->protocol))
->setPath(idx($config, 'path', $this->path))
->setVersion(idx($config, 'version', $this->version));
return $this;
}
public function getDisplayName() {
return pht('ElasticSearch');
}
public function getStatusViewColumns() {
return array(
pht('Protocol') => $this->getProtocol(),
pht('Host') => $this->getHost(),
pht('Port') => $this->getPort(),
pht('Index Path') => $this->getPath(),
pht('Elastic Version') => $this->getVersion(),
pht('Roles') => implode(', ', array_keys($this->getRoles())),
);
}
public function setProtocol($protocol) {
$this->protocol = $protocol;
return $this;
}
public function getProtocol() {
return $this->protocol;
}
public function setPath($path) {
$this->path = $path;
return $this;
}
public function getPath() {
return $this->path;
}
public function setVersion($version) {
$this->version = $version;
return $this;
}
public function getVersion() {
return $this->version;
}
public function getURI($to_path = null) {
$uri = id(new PhutilURI('http://'.$this->getHost()))
->setProtocol($this->getProtocol())
->setPort($this->getPort())
->setPath($this->getPath());
if ($to_path) {
$uri->appendPath($to_path);
}
return $uri;
}
public function getConnectionStatus() {
$status = $this->getEngine()->indexIsSane($this);
return $status ? parent::STATUS_OKAY : parent::STATUS_FAIL;
}
}

View file

@ -0,0 +1,34 @@
<?php
final class PhabricatorMySQLSearchHost
extends PhabricatorSearchHost {
public function setConfig($config) {
$this->setRoles(idx($config, 'roles',
array('read' => true, 'write' => true)));
return $this;
}
public function getDisplayName() {
return 'MySQL';
}
public function getStatusViewColumns() {
return array(
pht('Protocol') => 'mysql',
pht('Roles') => implode(', ', array_keys($this->getRoles())),
);
}
public function getProtocol() {
return 'mysql';
}
public function getConnectionStatus() {
PhabricatorDatabaseRef::queryAll();
$ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search');
$status = $ref->getConnectionStatus();
return $status;
}
}

View file

@ -0,0 +1,163 @@
<?php
abstract class PhabricatorSearchHost
extends Phobject {
const KEY_REFS = 'cluster.search.refs';
const KEY_HEALTH = 'cluster.search.health';
protected $engine;
protected $healthRecord;
protected $roles = array();
protected $disabled;
protected $host;
protected $port;
protected $hostRefs = array();
const STATUS_OKAY = 'okay';
const STATUS_FAIL = 'fail';
public function __construct(PhabricatorFulltextStorageEngine $engine) {
$this->engine = $engine;
}
public function setDisabled($disabled) {
$this->disabled = $disabled;
return $this;
}
public function getDisabled() {
return $this->disabled;
}
/**
* @return PhabricatorFulltextStorageEngine
*/
public function getEngine() {
return $this->engine;
}
public function isWritable() {
return $this->hasRole('write');
}
public function isReadable() {
return $this->hasRole('read');
}
public function hasRole($role) {
return isset($this->roles[$role]) && $this->roles[$role] === true;
}
public function setRoles(array $roles) {
foreach ($roles as $role => $val) {
$this->roles[$role] = $val;
}
return $this;
}
public function getRoles() {
$roles = array();
foreach ($this->roles as $key => $val) {
if ($val) {
$roles[$key] = $val;
}
}
return $roles;
}
public function setPort($value) {
$this->port = $value;
return $this;
}
public function getPort() {
return $this->port;
}
public function setHost($value) {
$this->host = $value;
return $this;
}
public function getHost() {
return $this->host;
}
public function getHealthRecordCacheKey() {
$host = $this->getHost();
$port = $this->getPort();
$key = self::KEY_HEALTH;
return "{$key}({$host}, {$port})";
}
/**
* @return PhabricatorClusterServiceHealthRecord
*/
public function getHealthRecord() {
if (!$this->healthRecord) {
$this->healthRecord = new PhabricatorClusterServiceHealthRecord(
$this->getHealthRecordCacheKey());
}
return $this->healthRecord;
}
public function didHealthCheck($reachable) {
$record = $this->getHealthRecord();
$should_check = $record->getShouldCheck();
if ($should_check) {
$record->didHealthCheck($reachable);
}
}
/**
* @return string[] Get a list of fields to show in the status overview UI
*/
abstract public function getStatusViewColumns();
abstract public function getConnectionStatus();
public static function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $doc) {
$services = self::getAllServices();
$indexed = 0;
foreach (self::getWritableHostForEachService() as $host) {
$host->getEngine()->reindexAbstractDocument($doc);
$indexed++;
}
if ($indexed == 0) {
throw new PhabricatorClusterNoHostForRoleException('write');
}
}
public static function executeSearch(PhabricatorSavedQuery $query) {
$services = self::getAllServices();
foreach ($services as $service) {
$hosts = $service->getAllHostsForRole('read');
// try all hosts until one succeeds
foreach ($hosts as $host) {
$last_exception = null;
try {
$res = $host->getEngine()->executeSearch($query);
// return immediately if we get results without an exception
$host->didHealthCheck(true);
return $res;
} catch (Exception $ex) {
// try each server in turn, only throw if none succeed
$last_exception = $ex;
$host->didHealthCheck(false);
}
}
}
if ($last_exception) {
throw $last_exception;
}
return $res;
}
}

View file

@ -0,0 +1,259 @@
<?php
class PhabricatorSearchService
extends Phobject {
const KEY_REFS = 'cluster.search.refs';
protected $config;
protected $disabled;
protected $engine;
protected $hosts = array();
protected $hostsConfig;
protected $hostType;
protected $roles = array();
const STATUS_OKAY = 'okay';
const STATUS_FAIL = 'fail';
public function __construct(PhabricatorFulltextStorageEngine $engine) {
$this->engine = $engine;
$this->hostType = $engine->getHostType();
}
/**
* @throws Exception
*/
public function newHost($config) {
$host = clone($this->hostType);
$host_config = $this->config + $config;
$host->setConfig($host_config);
$this->hosts[] = $host;
return $host;
}
public function getEngine() {
return $this->engine;
}
public function getDisplayName() {
return $this->hostType->getDisplayName();
}
public function getStatusViewColumns() {
return $this->hostType->getStatusViewColumns();
}
public function setConfig($config) {
$this->config = $config;
if (!isset($config['hosts'])) {
$config['hosts'] = array(
array(
'host' => idx($config, 'host'),
'port' => idx($config, 'port'),
'protocol' => idx($config, 'protocol'),
'roles' => idx($config, 'roles'),
),
);
}
foreach ($config['hosts'] as $host) {
$this->newHost($host);
}
}
public function getConfig() {
return $this->config;
}
public function setDisabled($disabled) {
$this->disabled = $disabled;
return $this;
}
public function getDisabled() {
return $this->disabled;
}
public static function getConnectionStatusMap() {
return array(
self::STATUS_OKAY => array(
'icon' => 'fa-exchange',
'color' => 'green',
'label' => pht('Okay'),
),
self::STATUS_FAIL => array(
'icon' => 'fa-times',
'color' => 'red',
'label' => pht('Failed'),
),
);
}
public function isWritable() {
return $this->hasRole('write');
}
public function isReadable() {
return $this->hasRole('read');
}
public function hasRole($role) {
return isset($this->roles[$role]) && $this->roles[$role] === true;
}
public function setRoles(array $roles) {
foreach ($roles as $role => $val) {
if ($val === false && isset($this->roles[$role])) {
unset($this->roles[$role]);
} else {
$this->roles[$role] = $val;
}
}
return $this;
}
public function getRoles() {
return $this->roles;
}
public function getPort() {
return idx($this->config, 'port');
}
public function getProtocol() {
return idx($this->config, 'protocol');
}
public function getVersion() {
return idx($this->config, 'version');
}
public function getHosts() {
return $this->hosts;
}
/**
* Get a random host reference with the specified role, skipping hosts which
* failed recent health checks.
* @throws PhabricatorClusterNoHostForRoleException if no healthy hosts match.
* @return PhabricatorSearchHost
*/
public function getAnyHostForRole($role) {
$hosts = $this->getAllHostsForRole($role);
shuffle($hosts);
foreach ($hosts as $host) {
$health = $host->getHealthRecord();
if ($health->getIsHealthy()) {
return $host;
}
}
throw new PhabricatorClusterNoHostForRoleException($role);
}
/**
* Get all configured hosts for this service which have the specified role.
* @return PhabricatorSearchHost[]
*/
public function getAllHostsForRole($role) {
$hosts = array();
foreach ($this->hosts as $host) {
if ($host->hasRole($role)) {
$hosts[] = $host;
}
}
return $hosts;
}
/**
* Get a reference to all configured fulltext search cluster services
* @return PhabricatorSearchService[]
*/
public static function getAllServices() {
$cache = PhabricatorCaches::getRequestCache();
$refs = $cache->getKey(self::KEY_REFS);
if (!$refs) {
$refs = self::newRefs();
$cache->setKey(self::KEY_REFS, $refs);
}
return $refs;
}
/**
* Load all valid PhabricatorFulltextStorageEngine subclasses
*/
public static function loadAllFulltextStorageEngines() {
return id(new PhutilClassMapQuery())
->setAncestorClass('PhabricatorFulltextStorageEngine')
->setUniqueMethod('getEngineIdentifier')
->execute();
}
/**
* Create instances of PhabricatorSearchService based on configuration
* @return PhabricatorSearchService[]
*/
public static function newRefs() {
$services = PhabricatorEnv::getEnvConfig('cluster.search');
$engines = self::loadAllFulltextStorageEngines();
$refs = array();
foreach ($services as $config) {
$engine = $engines[$config['type']];
$cluster = new self($engine);
$cluster->setConfig($config);
$engine->setService($cluster);
$refs[] = $cluster;
}
return $refs;
}
/**
* (re)index the document: attempt to pass the document to all writable
* fulltext search hosts
* @throws PhabricatorClusterNoHostForRoleException
*/
public static function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $doc) {
$indexed = 0;
foreach (self::getAllServices() as $service) {
$service->getEngine()->reindexAbstractDocument($doc);
$indexed++;
}
if ($indexed == 0) {
throw new PhabricatorClusterNoHostForRoleException('write');
}
}
/**
* Execute a full-text query and return a list of PHIDs of matching objects.
* @return string[]
* @throws PhutilAggregateException
*/
public static function executeSearch(PhabricatorSavedQuery $query) {
$services = self::getAllServices();
$exceptions = array();
foreach ($services as $service) {
$engine = $service->getEngine();
// try all hosts until one succeeds
try {
$res = $engine->executeSearch($query);
// return immediately if we get results without an exception
return $res;
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
throw new PhutilAggregateException('All search engines failed:',
$exceptions);
}
}