[NO CLUE WHAT I'M DOING] Add an Elasticsearch engine

Summary:
I have no idea what I'm doing, but here's part of an elasticsearch engine. These things work:

  - Indexing stuff (??)
  - Searching for text/type?
  - Reconstructing things??

All the complicated stuff doesn't work. I'm having a hard time figuring out the best way to model things because elasticsearch's documentation is not exactly the most complete or illuminating.

@amckinley, does this look sane-ish so far? Particularly, the /phabricator/<type>/<phid>/ URI scheme and how I've set up the relationships and fields in the documents?

How should I model the relationship and field queries? I want, like, an "equal" query but it seems like I've got "text" or "term" to work with and neither are exact match? And "term" doesn't consider PHIDs to be terms since they have hyphens in them?

I'll keep kind of slogging my way forward here but if you have valuable wisdom to share it would probably get me to a better end state much faster. The whole query construction phase is pretty much black magic to me.

Test Plan: nyancat

Reviewers: amckinley, vrana

Reviewed By: vrana

CC: jungejason, tuomaspelkonen, aran, 20after4, vrana

Differential Revision: https://secure.phabricator.com/D790

conf/default.conf.php

@@ -687,6 +687,10 @@ return array(
 
 // -- Search ---------------------------------------------------------------- //
 
+  // Phabricator supports Elastic Search; to use it, specify a host like
+  // 'http://elastic.example.com:9200/' here.
+  'search.elastic.host'     => null,
+
   // Phabricator uses a search engine selector to choose which search engine
   // to use when indexing and reconstructing documents, and when executing
   // queries. You can override the engine selector to provide a new selector

src/__phutil_library_map__.php

@@ -862,6 +862,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchDocumentIndexer' => 'applications/search/index/indexer/base',
     'PhabricatorSearchDocumentRelationship' => 'applications/search/storage/document/relationship',
     'PhabricatorSearchEngine' => 'applications/search/engine/base',
+    'PhabricatorSearchEngineElastic' => 'applications/search/engine/elastic',
     'PhabricatorSearchEngineMySQL' => 'applications/search/engine/mysql',
     'PhabricatorSearchEngineSelector' => 'applications/search/selector/base',
     'PhabricatorSearchField' => 'applications/search/constants/field',
@@ -1714,6 +1715,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchDocument' => 'PhabricatorSearchDAO',
     'PhabricatorSearchDocumentField' => 'PhabricatorSearchDAO',
     'PhabricatorSearchDocumentRelationship' => 'PhabricatorSearchDAO',
+    'PhabricatorSearchEngineElastic' => 'PhabricatorSearchEngine',
     'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
     'PhabricatorSearchIndexController' => 'PhabricatorSearchBaseController',
     'PhabricatorSearchManiphestIndexer' => 'PhabricatorSearchDocumentIndexer',

src/applications/search/controller/search/PhabricatorSearchController.php

@@ -226,8 +226,6 @@ final class PhabricatorSearchController
 
       $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
       $results = $engine->executeSearch($query);
-      $results = ipull($results, 'phid');
-
       $results = $pager->sliceResults($results);
 
       if (!$request->getInt('page')) {

src/applications/search/engine/elastic/PhabricatorSearchEngineElastic.php

@@ -0,0 +1,173 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
+
+  public function reindexAbstractDocument(
+    PhabricatorSearchAbstractDocument $doc) {
+
+    $type = $doc->getDocumentType();
+    $phid = $doc->getPHID();
+
+    $spec = array(
+      'phid'          => $phid,
+      'type'          => $type,
+      'title'         => $doc->getDocumentTitle(),
+      'dateCreated'   => date('c', $doc->getDocumentCreated()),
+      'dateModified'  => date('c', $doc->getDocumentModified()),
+      'field'         => array(),
+      'relationship'  => array(),
+    );
+
+    foreach ($doc->getFieldData() as $field) {
+      list($ftype, $corpus, $aux_phid) = $field;
+      $spec['field'][$ftype][] = array(
+        'corpus'  => $corpus,
+        'aux'     => $aux_phid,
+      );
+    }
+
+    foreach ($doc->getRelationshipData() as $relationship) {
+      list($rtype, $to_phid, $to_type, $time) = $relationship;
+      $spec['relationship'][$rtype][] = array(
+        'phid'      => $to_phid,
+        'phidType'  => $to_type,
+        'when'      => date('c', $time),
+      );
+    }
+
+    $this->executeRequest(
+      "/phabricator/{$type}/{$phid}/",
+      $spec,
+      $is_write = true);
+  }
+
+  public function reconstructDocument($phid) {
+
+    $response = $this->executeRequest(
+      '/phabricator/_search',
+      array(
+        'query' => array(
+          'ids' => array(
+            'values' => array(
+              $phid,
+            ),
+          ),
+        ),
+      ),
+      $is_write = false);
+
+    $hit = $response['hits']['hits'][0]['_source'];
+    if (!$hit) {
+      return null;
+    }
+
+    $doc = new PhabricatorSearchAbstractDocument();
+    $doc->setPHID($hit['phid']);
+    $doc->setDocumentType($hit['type']);
+    $doc->setDocumentTitle($hit['title']);
+    $doc->setDocumentCreated(strtotime($hit['dateCreated']));
+    $doc->setDocumentModified(strtotime($hit['dateModified']));
+
+    foreach ($hit['field'] as $ftype => $fdefs) {
+      foreach ($fdefs as $fdef) {
+        $doc->addField(
+          $ftype,
+          $fdef['corpus'],
+          $fdef['aux']);
+      }
+    }
+
+    foreach ($hit['relationship'] as $rtype => $rships) {
+      foreach ($rships as $rship) {
+        $doc->addRelationship(
+          $rtype,
+          $rship['phid'],
+          $rship['phidType'],
+          strtotime($rship['when']));
+      }
+    }
+
+    return $doc;
+  }
+
+  public function executeSearch(PhabricatorSearchQuery $query) {
+
+    $spec = array(
+      'text' => array(
+        '_all' => $query->getQuery(),
+      ),
+    );
+
+    $type = $query->getParameter('type');
+    if ($type) {
+      $uri = "/phabricator/{$type}/_search";
+    } else {
+      $uri = "/phabricator/_search";
+    }
+
+    $response = $this->executeRequest(
+      $uri,
+      array(
+        'query' => $spec,
+      ),
+      $is_write = false);
+
+    $phids = array();
+    foreach ($response['hits']['hits'] as $hit) {
+      $phids[] = $hit['_id'];
+    }
+
+    return $phids;
+  }
+
+  private function executeRequest($path, array $data, $is_write) {
+    $uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
+    $uri = new PhutilURI($uri);
+    $data = json_encode($data);
+
+    $uri->setPath($path);
+
+    $protocol = $uri->getProtocol();
+    if ($protocol == 'https') {
+      $future = new HTTPSFuture($uri, $data);
+    } else {
+      $future = new HTTPFuture($uri, $data);
+    }
+
+    if ($is_write) {
+      $future->setMethod('PUT');
+    } else {
+      $future->setMethod('GET');
+    }
+
+    list($body) = $future->resolvex();
+
+    if ($is_write) {
+      return null;
+    }
+
+    $body = json_decode($body, true);
+    if (!is_array($body)) {
+      throw new Exception("elasticsearch server returned invalid JSON!");
+    }
+
+    return $body;
+  }
+
+}

src/applications/search/engine/elastic/__init__.php

@@ -0,0 +1,18 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/search/engine/base');
+phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
+phutil_require_module('phabricator', 'infrastructure/env');
+
+phutil_require_module('phutil', 'future/http/http');
+phutil_require_module('phutil', 'future/http/https');
+phutil_require_module('phutil', 'parser/uri');
+
+
+phutil_require_source('PhabricatorSearchEngineElastic.php');

src/applications/search/engine/mysql/PhabricatorSearchEngineMySQL.php

@@ -289,9 +289,6 @@ final class PhabricatorSearchEngineMySQL extends PhabricatorSearchEngine {
       $conn_r,
       'SELECT
           document.phid,
-          document.documentType,
-          document.documentTitle,
-          document.documentCreated
         FROM %T document
           %Q
           %Q
@@ -305,7 +302,7 @@ final class PhabricatorSearchEngineMySQL extends PhabricatorSearchEngine {
       $offset,
       $limit);
 
-    return $hits;
+    return ipull($hits, 'phid');
   }
 
   protected function joinRelationship($conn, $query, $field, $type) {

src/applications/search/selector/default/PhabricatorDefaultSearchEngineSelector.php

@@ -23,6 +23,9 @@ final class PhabricatorDefaultSearchEngineSelector
   extends PhabricatorSearchEngineSelector {
 
   public function newEngine() {
+    if (PhabricatorEnv::getEnvConfig('search.elastic.host')) {
+      return new PhabricatorSearchEngineElastic();
+    }
     return new PhabricatorSearchEngineMySQL();
   }
 }

src/applications/search/selector/default/__init__.php

@@ -6,8 +6,10 @@
 
 
 
+phutil_require_module('phabricator', 'applications/search/engine/elastic');
 phutil_require_module('phabricator', 'applications/search/engine/mysql');
 phutil_require_module('phabricator', 'applications/search/selector/base');
+phutil_require_module('phabricator', 'infrastructure/env');
 
 
 phutil_require_source('PhabricatorDefaultSearchEngineSelector.php');

src/applications/search/view/searchresult/PhabricatorSearchResultView.php

@@ -82,10 +82,19 @@ final class PhabricatorSearchResultView extends AphrontView {
         break;
     }
 
+    $index_link = phutil_render_tag(
+      'a',
+      array(
+        'href' => '/search/index/'.$handle->getPHID().'/',
+        'style' => 'float: right',
+      ),
+      'Examine Index');
+
     return
       '<div class="phabricator-search-result">'.
         $img.
         '<div class="result-desc">'.
+          $index_link.
           phutil_render_tag(
             'a',
             array(