Properly create Elasticsearch index

Summary:
When the index does not exist and auto_create_index isn't
enabled, running ./bin/index results in a failure. That's
T5990

Instead create an index properly. This also allows us to do
nice things like do a proper mapping and analysis like for
substring matching like outlined by @fabe in T6552.

Test Plan:
Deleted and created index multiple times to verify
proper index creation and usage.

Reviewers: epriestley, #blessed_reviewers

Reviewed By: epriestley, #blessed_reviewers

Subscribers: Korvin, manybubbles, chasemp, fabe, epriestley

Differential Revision: https://secure.phabricator.com/D10955

src/__phutil_library_map__.php

@@ -2310,6 +2310,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
     'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php',
     'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
+    'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
     'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
     'PhabricatorSearchOrderController' => 'applications/search/controller/PhabricatorSearchOrderController.php',
     'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php',
@@ -2349,6 +2350,7 @@ phutil_register_library_map(array(
     'PhabricatorSetupCheckBinaries' => 'applications/config/check/PhabricatorSetupCheckBinaries.php',
     'PhabricatorSetupCheckDaemons' => 'applications/config/check/PhabricatorSetupCheckDaemons.php',
     'PhabricatorSetupCheckDatabase' => 'applications/config/check/PhabricatorSetupCheckDatabase.php',
+    'PhabricatorSetupCheckElastic' => 'applications/config/check/PhabricatorSetupCheckElastic.php',
     'PhabricatorSetupCheckExtensions' => 'applications/config/check/PhabricatorSetupCheckExtensions.php',
     'PhabricatorSetupCheckExtraConfig' => 'applications/config/check/PhabricatorSetupCheckExtraConfig.php',
     'PhabricatorSetupCheckFileinfo' => 'applications/config/check/PhabricatorSetupCheckFileinfo.php',
@@ -5515,6 +5517,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
     'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
     'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
+    'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
     'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorSearchOrderController' => 'PhabricatorSearchBaseController',
     'PhabricatorSearchResultView' => 'AphrontView',
@@ -5551,6 +5554,7 @@ phutil_register_library_map(array(
     'PhabricatorSetupCheckBinaries' => 'PhabricatorSetupCheck',
     'PhabricatorSetupCheckDaemons' => 'PhabricatorSetupCheck',
     'PhabricatorSetupCheckDatabase' => 'PhabricatorSetupCheck',
+    'PhabricatorSetupCheckElastic' => 'PhabricatorSetupCheck',
     'PhabricatorSetupCheckExtensions' => 'PhabricatorSetupCheck',
     'PhabricatorSetupCheckExtraConfig' => 'PhabricatorSetupCheck',
     'PhabricatorSetupCheckFileinfo' => 'PhabricatorSetupCheck',

src/applications/config/check/PhabricatorSetupCheckElastic.php

@@ -0,0 +1,39 @@
+<?php
+
+final class PhabricatorSetupCheckElastic extends PhabricatorSetupCheck {
+
+  protected function executeChecks() {
+    if (PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
+      $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
+      if (!$engine->indexExists()) {
+        $summary = pht(
+          'You enabled Elasticsearch but the index does not exist.');
+
+        $message = pht(
+          'You likely enabled search.elastic.host without creating the '.
+          'index. Run `./bin/search init` to correct the index.');
+
+        $this
+          ->newIssue('elastic.missing-index')
+          ->setName(pht('Elasticsearch index Not Found'))
+          ->setSummary($summary)
+          ->setMessage($message)
+          ->addRelatedPhabricatorConfig('search.elastic.host');
+      } else if (!$engine->indexIsSane()) {
+        $summary = pht(
+          'Elasticsearch index exists but needs correction.');
+
+        $message = pht(
+          'Either the Phabricator schema for Elasticsearch has changed '.
+          'or Elasticsearch created the index automatically. Run '.
+          '`./bin/search init` to correct the index.');
+
+        $this
+          ->newIssue('elastic.broken-index')
+          ->setName(pht('Elasticsearch index Incorrect'))
+          ->setSummary($summary)
+          ->setMessage($message);
+      }
+    }
+  }
+}

src/applications/search/engine/PhabricatorSearchEngine.php

@@ -36,4 +36,26 @@ abstract class PhabricatorSearchEngine {
    */
   abstract public function executeSearch(PhabricatorSavedQuery $query);
 
+  /**
+   * Does the search index exist?
+   *
+   * @return bool
+   */
+  abstract public function indexExists();
+
+  /**
+   * Is the index in a usable state?
+   *
+   * @return bool
+   */
+  public function indexIsSane() {
+    return $this->indexExists();
+  }
+
+  /**
+   * Do any sort of setup for the search index
+   *
+   * @return void
+   */
+  public function initIndex() {}
 }

src/applications/search/engine/PhabricatorSearchEngineElastic.php

@@ -52,10 +52,7 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
       );
     }
 
-    $this->executeRequest(
-      "/{$type}/{$phid}/",
-      $spec,
-      $is_write = true);
+    $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT');
   }
 
   public function reconstructDocument($phid) {
@@ -236,22 +233,146 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
     return $phids;
   }
 
-  private function executeRequest($path, array $data, $is_write = false) {
+  public function indexExists() {
+    try {
+      return (bool)$this->executeRequest('/_status/', array());
+    } catch (HTTPFutureHTTPResponseStatus $e) {
+      if ($e->getStatusCode() == 404) {
+        return false;
+      }
+      throw $e;
+    }
+  }
+
+  private function getIndexConfiguration() {
+    $data = array();
+    $data['settings'] = array(
+      'index' => array(
+        'auto_expand_replicas' => '0-2',
+        'analysis' => array(
+          'filter' => array(
+            'trigrams_filter' => array(
+              'min_gram' => 3,
+              'type' => 'ngram',
+              'max_gram' => 3,
+            ),
+          ),
+          'analyzer' => array(
+            'custom_trigrams' => array(
+              'type' => 'custom',
+              'filter' => array(
+                'lowercase',
+                'kstem',
+                'trigrams_filter',
+              ),
+              'tokenizer' => 'standard',
+            ),
+          ),
+        ),
+      ),
+    );
+
+    $types = array_keys(
+      PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes());
+    foreach ($types as $type) {
+      $data['mappings'][$type]['properties']['field']['properties']['corpus'] =
+        array( 'type' => 'string', 'analyzer' => 'custom_trigrams' );
+    }
+
+    return $data;
+  }
+
+  public function indexIsSane() {
+    if (!$this->indexExists()) {
+      return false;
+    }
+
+    $cur_mapping = $this->executeRequest('/_mapping/', array());
+    $cur_settings = $this->executeRequest('/_settings/', array());
+    $actual = array_merge($cur_settings[$this->index],
+      $cur_mapping[$this->index]);
+
+    return $this->check($actual, $this->getIndexConfiguration());
+  }
+
+  /**
+   * Recursively check if two Elasticsearch configuration arrays are equal
+   *
+   * @param $actual
+   * @param $required array
+   * @return bool
+   */
+  private function check($actual, $required) {
+    foreach ($required as $key => $value) {
+      if (!array_key_exists($key, $actual)) {
+        if ($key === '_all') {
+          // The _all field never comes back so we just have to assume it
+          // is set correctly.
+          continue;
+        }
+        return false;
+      }
+      if (is_array($value)) {
+        if (!is_array($actual[$key])) {
+          return false;
+        }
+        if (!$this->check($actual[$key], $value)) {
+          return false;
+        }
+        continue;
+      }
+
+      $actual[$key] = self::normalizeConfigValue($actual[$key]);
+      $value = self::normalizeConfigValue($value);
+      if ($actual[$key] != $value) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Normalize a config value for comparison. Elasticsearch accepts all kinds
+   * of config values but it tends to throw back 'true' for true and 'false' for
+   * false so we normalize everything. Sometimes, oddly, it'll throw back false
+   * for false....
+   *
+   * @param mixed $value config value
+   * @return mixed value normalized
+   */
+  private static function normalizeConfigValue($value) {
+    if ($value === true) {
+      return 'true';
+    } else if ($value === false) {
+      return 'false';
+    }
+    return $value;
+  }
+
+  public function initIndex() {
+    if ($this->indexExists()) {
+      $this->executeRequest('/', array(), 'DELETE');
+    }
+    $data = $this->getIndexConfiguration();
+    $this->executeRequest('/', $data, 'PUT');
+  }
+
+  private function executeRequest($path, array $data, $method = 'GET') {
     $uri = new PhutilURI($this->uri);
     $uri->setPath($this->index);
     $uri->appendPath($path);
     $data = json_encode($data);
 
     $future = new HTTPSFuture($uri, $data);
-    if ($is_write) {
-      $future->setMethod('PUT');
+    if ($method != 'GET') {
+      $future->setMethod($method);
     }
     if ($this->getTimeout()) {
       $future->setTimeout($this->getTimeout());
     }
     list($body) = $future->resolvex();
 
-    if ($is_write) {
+    if ($method != 'GET') {
       return null;
     }
 

src/applications/search/engine/PhabricatorSearchEngineMySQL.php

@@ -331,4 +331,7 @@ final class PhabricatorSearchEngineMySQL extends PhabricatorSearchEngine {
     return $sql;
   }
 
+  public function indexExists() {
+    return true;
+  }
 }

src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php

@@ -0,0 +1,50 @@
+<?php
+
+final class PhabricatorSearchManagementInitWorkflow
+  extends PhabricatorSearchManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('init')
+      ->setSynopsis('Initialize or repair an index.')
+      ->setExamples('**init**');
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $console = PhutilConsole::getConsole();
+
+    $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
+
+    $work_done = false;
+    if (!$engine->indexExists()) {
+      $console->writeOut(
+        '%s',
+        pht('Index does not exist, creating...'));
+      $engine->initIndex();
+      $console->writeOut(
+        "%s\n",
+        pht('done.'));
+      $work_done = true;
+    } else if (!$engine->indexIsSane()) {
+      $console->writeOut(
+        '%s',
+          pht('Index exists but is incorrect, fixing...'));
+      $engine->initIndex();
+      $console->writeOut(
+        "%s\n",
+        pht('done.'));
+      $work_done = true;
+    }
+
+    if ($work_done) {
+      $console->writeOut(
+        "%s\n",
+        pht('Index maintenance complete. Run `./bin/search index` to '.
+            'reindex documents'));
+    } else {
+      $console->writeOut(
+        "%s\n",
+        pht('Nothing to do.'));
+    }
+  }
+}