Improve elasticsearch

Summary: I thought that this will be fun but the elasticsearch API is horrible and the documentation is poor.

Test Plan:
Search for:

- string
- author
- author, owner
- string, author
- open
- string, open, author
- string, exclude
- several authors, several owners
- nothing
- probably all other combinations

Normally, such an exhaustive test plan wouldn't be required but each combination requires a completely different query.

Reviewers: epriestley, jungejason

Reviewed By: epriestley

CC: aran, Koolvin, btrahan

Differential Revision: https://secure.phabricator.com/D2298

src/applications/search/controller/search/PhabricatorSearchController.php

@@ -113,16 +113,9 @@ final class PhabricatorSearchController
       }
     }
 
-    $more = PhabricatorEnv::getEnvConfig('search.more-document-types', array());
-
     $options = array(
       '' => 'All Documents',
-      PhabricatorPHIDConstants::PHID_TYPE_DREV => 'Differential Revisions',
+    ) + PhabricatorSearchAbstractDocument::getSupportedTypes();
-      PhabricatorPHIDConstants::PHID_TYPE_CMIT => 'Repository Commits',
-      PhabricatorPHIDConstants::PHID_TYPE_TASK => 'Maniphest Tasks',
-      PhabricatorPHIDConstants::PHID_TYPE_WIKI => 'Phriction Documents',
-      PhabricatorPHIDConstants::PHID_TYPE_USER => 'Phabricator Users',
-    ) + $more;
 
     $status_options = array(
       0 => 'Open and Closed Documents',

src/applications/search/controller/search/__init__.php

@@ -18,11 +18,11 @@ phutil_require_module('phabricator', 'applications/repository/storage/repository
 phutil_require_module('phabricator', 'applications/search/constants/scope');
 phutil_require_module('phabricator', 'applications/search/controller/base');
 phutil_require_module('phabricator', 'applications/search/engine/jumpnav');
+phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
 phutil_require_module('phabricator', 'applications/search/selector/base');
 phutil_require_module('phabricator', 'applications/search/storage/query');
 phutil_require_module('phabricator', 'applications/search/view/searchresult');
 phutil_require_module('phabricator', 'infrastructure/celerity/api');
-phutil_require_module('phabricator', 'infrastructure/env');
 phutil_require_module('phabricator', 'view/control/pager');
 phutil_require_module('phabricator', 'view/form/base');
 phutil_require_module('phabricator', 'view/form/control/select');

src/applications/search/engine/elastic/PhabricatorSearchEngineElastic.php

@@ -1,7 +1,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,23 +23,19 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
 
     $type = $doc->getDocumentType();
     $phid = $doc->getPHID();
+    $handle = PhabricatorObjectHandleData::loadOneHandle($phid);
 
     $spec = array(
-      'phid'          => $phid,
-      'type'          => $type,
       'title'         => $doc->getDocumentTitle(),
-      'dateCreated'   => date('c', $doc->getDocumentCreated()),
+      'url'           => PhabricatorEnv::getProductionURI($handle->getURI()),
-      'dateModified'  => date('c', $doc->getDocumentModified()),
+      'dateCreated'   => $doc->getDocumentCreated(),
+      '_timestamp'    => $doc->getDocumentModified(),
       'field'         => array(),
       'relationship'  => array(),
     );
 
     foreach ($doc->getFieldData() as $field) {
-      list($ftype, $corpus, $aux_phid) = $field;
+      $spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field);
-      $spec['field'][$ftype][] = array(
-        'corpus'  => $corpus,
-        'aux'     => $aux_phid,
-      );
     }
 
     foreach ($doc->getRelationshipData() as $relationship) {
@@ -47,7 +43,7 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
       $spec['relationship'][$rtype][] = array(
         'phid'      => $to_phid,
         'phidType'  => $to_type,
-        'when'      => date('c', $time),
+        'when'      => $time,
       );
     }
 
@@ -58,39 +54,25 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
   }
 
   public function reconstructDocument($phid) {
+    $type = phid_get_type($phid);
 
-    $response = $this->executeRequest(
+    $response = $this->executeRequest("/phabricator/{$type}/{$phid}", array());
-      '/phabricator/_search',
-      array(
-        'query' => array(
-          'ids' => array(
-            'values' => array(
-              $phid,
-            ),
-          ),
-        ),
-      ),
-      $is_write = false);
 
-    $hit = $response['hits']['hits'][0]['_source'];
+    if (empty($response['exists'])) {
-    if (!$hit) {
       return null;
     }
 
-    $doc = new PhabricatorSearchAbstractDocument();
+    $hit = $response['_source'];
-    $doc->setPHID($hit['phid']);
-    $doc->setDocumentType($hit['type']);
-    $doc->setDocumentTitle($hit['title']);
-    $doc->setDocumentCreated(strtotime($hit['dateCreated']));
-    $doc->setDocumentModified(strtotime($hit['dateModified']));
 
-    foreach ($hit['field'] as $ftype => $fdefs) {
+    $doc = new PhabricatorSearchAbstractDocument();
-      foreach ($fdefs as $fdef) {
+    $doc->setPHID($phid);
-        $doc->addField(
+    $doc->setDocumentType($response['_type']);
-          $ftype,
+    $doc->setDocumentTitle($hit['title']);
-          $fdef['corpus'],
+    $doc->setDocumentCreated($hit['dateCreated']);
-          $fdef['aux']);
+    $doc->setDocumentModified($hit['_timestamp']);
-      }
+
+    foreach ($hit['field'] as $fdef) {
+      $doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']);
     }
 
     foreach ($hit['relationship'] as $rtype => $rships) {
@@ -99,7 +81,7 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
           $rtype,
           $rship['phid'],
           $rship['phidType'],
-          strtotime($rship['when']));
+          $rship['when']);
       }
     }
 
@@ -108,35 +90,96 @@ final class PhabricatorSearchEngineElastic extends PhabricatorSearchEngine {
 
   public function executeSearch(PhabricatorSearchQuery $query) {
 
-    $spec = array(
+    $spec = array();
-      'text' => array(
+    $filter = array();
-        '_all' => $query->getQuery(),
+
+    if ($query->getQuery()) {
+      $spec[] = array(
+        'field' => array(
+          'field.corpus' => $query->getQuery(),
         ),
       );
+    }
+
+    $exclude = $query->getParameter('exclude');
+    if ($exclude) {
+      $filter[] = array(
+        'not' => array(
+          'ids' => array(
+            'values' => array($exclude),
+          ),
+        ),
+      );
+    }
 
     $type = $query->getParameter('type');
     if ($type) {
       $uri = "/phabricator/{$type}/_search";
     } else {
-      $uri = "/phabricator/_search";
+      // Don't use '/phabricator/_search' for the case that there is something
+      // else in the index (for example if 'phabricator' is only an alias to
+      // some bigger index).
+      $types = PhabricatorSearchAbstractDocument::getSupportedTypes();
+      $uri = '/phabricator/' . implode(',', array_keys($types)) . '/_search';
     }
 
-    $response = $this->executeRequest(
+    $rel_mapping = array(
-      $uri,
+      'author' => PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR,
-      array(
+      'open' => PhabricatorSearchRelationship::RELATIONSHIP_OPEN,
-        'query' => $spec,
+      'owner' => PhabricatorSearchRelationship::RELATIONSHIP_OWNER,
+      'project' => PhabricatorSearchRelationship::RELATIONSHIP_PROJECT,
+      'repository' => PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY,
+    );
+    foreach ($rel_mapping as $name => $field) {
+      $param = $query->getParameter($name);
+      if (is_array($param)) {
+        $should = array();
+        foreach ($param as $val) {
+          $should[] = array(
+            'text' => array(
+              "relationship.{$field}.phid" => array(
+                'query' => $val,
+                'type' => 'phrase',
               ),
-      $is_write = false);
+            ),
-
+          );
-    $phids = array();
+        }
-    foreach ($response['hits']['hits'] as $hit) {
+        $spec[] = array('bool' => array('should' => $should));
-      $phids[] = $hit['_id'];
+      } else if ($param) {
+        $filter[] = array(
+          'exists' => array(
+            'field' => "relationship.{$field}.phid",
+          ),
+        );
+      }
     }
 
+    if ($spec) {
+      $spec = array('query' => array('bool' => array('must' => $spec)));
+    }
+
+    if ($filter) {
+      $filter = array('filter' => array('and' => $filter));
+      if ($spec) {
+        $spec = array(
+          'query' => array(
+            'filtered' => $spec + $filter,
+          ),
+        );
+      } else {
+        $spec = $filter;
+      }
+    }
+
+    $spec['from'] = (int)$query->getParameter('offset', 0);
+    $spec['size'] = (int)$query->getParameter('limit', 0);
+    $response = $this->executeRequest($uri, $spec);
+
+    $phids = ipull($response['hits']['hits'], '_id');
     return $phids;
   }
 
-  private function executeRequest($path, array $data, $is_write) {
+  private function executeRequest($path, array $data, $is_write = false) {
     $uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
     $uri = new PhutilURI($uri);
     $data = json_encode($data);

src/applications/search/engine/elastic/__init__.php

@@ -6,6 +6,9 @@
 
 
 
+phutil_require_module('phabricator', 'applications/phid/handle/data');
+phutil_require_module('phabricator', 'applications/phid/utils');
+phutil_require_module('phabricator', 'applications/search/constants/relationship');
 phutil_require_module('phabricator', 'applications/search/engine/base');
 phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
 phutil_require_module('phabricator', 'infrastructure/env');
@@ -13,6 +16,7 @@ phutil_require_module('phabricator', 'infrastructure/env');
 phutil_require_module('phutil', 'future/http/http');
 phutil_require_module('phutil', 'future/http/https');
 phutil_require_module('phutil', 'parser/uri');
+phutil_require_module('phutil', 'utils');
 
 
 phutil_require_source('PhabricatorSearchEngineElastic.php');

src/applications/search/engine/mysql/PhabricatorSearchEngineMySQL.php

@@ -305,7 +305,12 @@ final class PhabricatorSearchEngineMySQL extends PhabricatorSearchEngine {
     return ipull($hits, 'phid');
   }
 
-  protected function joinRelationship($conn, $query, $field, $type) {
+  protected function joinRelationship(
+    AphrontDatabaseConnection $conn,
+    PhabricatorSearchQuery $query,
+    $field,
+    $type) {
+
     $phids = $query->getParameter($field, array());
     if (!$phids) {
       return null;

src/applications/search/index/abstractdocument/PhabricatorSearchAbstractDocument.php

@@ -1,7 +1,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,6 +29,17 @@ final class PhabricatorSearchAbstractDocument {
   private $fields = array();
   private $relationships = array();
 
+  public static function getSupportedTypes() {
+    $more = PhabricatorEnv::getEnvConfig('search.more-document-types', array());
+    return array(
+      PhabricatorPHIDConstants::PHID_TYPE_DREV => 'Differential Revisions',
+      PhabricatorPHIDConstants::PHID_TYPE_CMIT => 'Repository Commits',
+      PhabricatorPHIDConstants::PHID_TYPE_TASK => 'Maniphest Tasks',
+      PhabricatorPHIDConstants::PHID_TYPE_WIKI => 'Phriction Documents',
+      PhabricatorPHIDConstants::PHID_TYPE_USER => 'Phabricator Users',
+    ) + $more;
+  }
+
   public function setPHID($phid) {
     $this->phid = $phid;
     return $this;

src/applications/search/index/abstractdocument/__init__.php

@@ -6,7 +6,9 @@
 
 
 
+phutil_require_module('phabricator', 'applications/phid/constants');
 phutil_require_module('phabricator', 'applications/search/constants/field');
+phutil_require_module('phabricator', 'infrastructure/env');
 
 
 phutil_require_source('PhabricatorSearchAbstractDocument.php');