Generate expected schemata for Search

Summary:
Ref T1191. Notable:

  - Drops a very old saved query table. See comments inline: plan was to remove it after a year. It's been ~a year and two weeks.
  - This has our only fulltext index. I'm not supporting that formally for now, but left a note.
  - This has our only MyISAM table. I'm not supporting that explicitly for now, but it shouldn't affect anything. I may deal with this in the future.
  - These tables don't actually write directly via Lisk, so there's some fiddling to get the schemata right.

Test Plan: Down to ~250 warnings. No more surplus databases or tables.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T1191

Differential Revision: https://secure.phabricator.com/D10589

resources/sql/autopatches/20140927.schema.01.dropsearchq.sql

@@ -0,0 +1 @@
+DROP TABLE {$NAMESPACE}_search.search_query;

resources/sql/patches/20130913.maniphest.1.migratesearch.php

@@ -1,212 +1,5 @@
 <?php
 
-// NOTE: If you need to make any significant updates to this to deal with
+// This was a complex migration that modernized old search queries. It was
-// future changes to objects, it's probably better to just wipe the whole
+// removed after giving installs a year to perform it because it would be
-// migration. This feature doesn't see overwhelming amounts of use, and users
+// unreasonably complex to maintain and users can easily recreate queries.
-// who do use it can recreate their queries fairly easily with the new
-// interface. By the time this needs to be updated, the vast majority of
-// users who it impacts will likely have migrated their data already.
-
-$table = new ManiphestTask();
-$conn_w = $table->establishConnection('w');
-
-$search_table = new PhabricatorSearchQuery();
-$search_conn_w = $search_table->establishConnection('w');
-
-// See T1812. This is an old status constant from the time of this migration.
-$old_open_status = 0;
-
-echo "Updating saved Maniphest queries...\n";
-$rows = new LiskRawMigrationIterator($conn_w, 'maniphest_savedquery');
-foreach ($rows as $row) {
-  $id = $row['id'];
-  echo "Updating query {$id}...\n";
-
-  $data = queryfx_one(
-    $search_conn_w,
-    'SELECT parameters FROM %T WHERE queryKey = %s',
-    $search_table->getTableName(),
-    $row['queryKey']);
-  if (!$data) {
-    echo "Unable to locate query data.\n";
-    continue;
-  }
-
-  $data = json_decode($data['parameters'], true);
-  if (!is_array($data)) {
-    echo "Unable to decode query data.\n";
-    continue;
-  }
-
-  if (idx($data, 'view') != 'custom') {
-    echo "Query is not a custom query.\n";
-    continue;
-  }
-
-  $new_data = array(
-    'limit' => 1000,
-  );
-
-  if (isset($data['lowPriority']) || isset($data['highPriority'])) {
-    $lo = idx($data, 'lowPriority');
-    $hi = idx($data, 'highPriority');
-
-    $priorities = array();
-    $all = ManiphestTaskPriority::getTaskPriorityMap();
-    foreach ($all as $pri => $name) {
-      if (($lo !== null) && ($pri < $lo)) {
-        continue;
-      }
-      if (($hi !== null) && ($pri > $hi)) {
-        continue;
-      }
-      $priorities[] = $pri;
-    }
-
-    if (count($priorities) != count($all)) {
-      $new_data['priorities'] = $priorities;
-    }
-  }
-
-  foreach ($data as $key => $value) {
-    switch ($key) {
-      case 'fullTextSearch':
-        if (strlen($value)) {
-          $new_data['fulltext'] = $value;
-        }
-        break;
-      case 'userPHIDs':
-        // This was (I think?) one-off data provied to specific hard-coded
-        // queries.
-        break;
-      case 'projectPHIDs':
-        foreach ($value as $k => $v) {
-          if ($v === null || $v === ManiphestTaskOwner::PROJECT_NO_PROJECT) {
-            $new_data['withNoProject'] = true;
-            unset($value[$k]);
-            break;
-          }
-        }
-        if ($value) {
-          $new_data['allProjectPHIDs'] = $value;
-        }
-        break;
-      case 'anyProjectPHIDs':
-        if ($value) {
-          $new_data['anyProjectPHIDs'] = $value;
-        }
-        break;
-      case 'anyUserProjectPHIDs':
-        if ($value) {
-          $new_data['userProjectPHIDs'] = $value;
-        }
-        break;
-      case 'excludeProjectPHIDs':
-        if ($value) {
-          $new_data['excludeProjectPHIDs'] = $value;
-        }
-        break;
-      case 'ownerPHIDs':
-        foreach ($value as $k => $v) {
-          if ($v === null || $v === ManiphestTaskOwner::OWNER_UP_FOR_GRABS) {
-            $new_data['withUnassigned'] = true;
-            unset($value[$k]);
-            break;
-          }
-        }
-        if ($value) {
-          $new_data['assignedPHIDs'] = $value;
-        }
-        break;
-      case 'authorPHIDs':
-        if ($value) {
-          $new_data['authorPHIDs'] = $value;
-        }
-        break;
-      case 'taskIDs':
-        if ($value) {
-          $new_data['ids'] = $value;
-        }
-        break;
-      case 'status':
-        $include_open = !empty($value['open']);
-        $include_closed = !empty($value['closed']);
-
-        if ($include_open xor $include_closed) {
-          if ($include_open) {
-            $new_data['statuses'] = array(
-              $old_open_status,
-            );
-          } else {
-            $statuses = array();
-            foreach (ManiphestTaskStatus::getTaskStatusMap() as $status => $n) {
-              if ($status != $old_open_status) {
-                $statuses[] = $status;
-              }
-            }
-            $new_data['statuses'] = $statuses;
-          }
-        }
-        break;
-      case 'order':
-        $map = array(
-          'priority' => 'priority',
-          'updated' => 'updated',
-          'created' => 'created',
-          'title' => 'title',
-        );
-        if (isset($map[$value])) {
-          $new_data['order'] = $map[$value];
-        } else {
-          $new_data['order'] = 'priority';
-        }
-        break;
-      case 'group':
-        $map = array(
-          'priority' => 'priority',
-          'owner' => 'assigned',
-          'status' => 'status',
-          'project' => 'project',
-          'none' => 'none',
-        );
-        if (isset($map[$value])) {
-          $new_data['group'] = $map[$value];
-        } else {
-          $new_data['group'] = 'priority';
-        }
-        break;
-    }
-  }
-
-  $saved = id(new PhabricatorSavedQuery())
-    ->setEngineClassName('ManiphestTaskSearchEngine');
-
-  foreach ($new_data as $key => $value) {
-    $saved->setParameter($key, $value);
-  }
-
-  try {
-    $saved->save();
-  } catch (AphrontDuplicateKeyQueryException $ex) {
-    // Ignore this, we just have duplicate saved queries.
-  }
-
-  $named = id(new PhabricatorNamedQuery())
-    ->setEngineClassName('ManiphestTaskSearchEngine')
-    ->setQueryKey($saved->getQueryKey())
-    ->setQueryName($row['name'])
-    ->setUserPHID($row['userPHID']);
-
-  try {
-    $named->save();
-  } catch (Exception $ex) {
-    // The user already has this query under another name. This can occur if
-    // the migration runs twice.
-    echo "Failed to save named query.\n";
-    continue;
-  }
-
-  echo "OK.\n";
-}
-
-echo "Done.\n";

src/__phutil_library_map__.php

@@ -2213,9 +2213,9 @@ phutil_register_library_map(array(
     'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
     'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
     'PhabricatorSearchOrderController' => 'applications/search/controller/PhabricatorSearchOrderController.php',
-    'PhabricatorSearchQuery' => 'applications/search/storage/PhabricatorSearchQuery.php',
     'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php',
     'PhabricatorSearchResultView' => 'applications/search/view/PhabricatorSearchResultView.php',
+    'PhabricatorSearchSchemaSpec' => 'applications/search/storage/PhabricatorSearchSchemaSpec.php',
     'PhabricatorSearchSelectController' => 'applications/search/controller/PhabricatorSearchSelectController.php',
     'PhabricatorSearchWorker' => 'applications/search/worker/PhabricatorSearchWorker.php',
     'PhabricatorSecurityConfigOptions' => 'applications/config/option/PhabricatorSecurityConfigOptions.php',
@@ -5216,8 +5216,8 @@ phutil_register_library_map(array(
     'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
     'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorSearchOrderController' => 'PhabricatorSearchBaseController',
-    'PhabricatorSearchQuery' => 'PhabricatorSearchDAO',
     'PhabricatorSearchResultView' => 'AphrontView',
+    'PhabricatorSearchSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'PhabricatorSearchSelectController' => 'PhabricatorSearchBaseController',
     'PhabricatorSearchWorker' => 'PhabricatorWorker',
     'PhabricatorSecurityConfigOptions' => 'PhabricatorApplicationConfigOptions',

src/applications/search/storage/PhabricatorNamedQuery.php

@@ -12,6 +12,25 @@ final class PhabricatorNamedQuery extends PhabricatorSearchDAO
   protected $isDisabled = 0;
   protected $sequence   = 0;
 
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'engineClassName' => 'text128',
+        'queryName' => 'text255',
+        'queryKey' => 'bytes12',
+        'isBuiltin' => 'bool',
+        'isDisabled' => 'bool',
+        'sequence' => 'uint32',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_userquery' => array(
+          'columns' => array('userPHID', 'engineClassName', 'queryKey'),
+          'unique' => true,
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
   public function getSortKey() {
     return sprintf('~%010d%010d', $this->sequence, $this->getID());
   }

src/applications/search/storage/PhabricatorSavedQuery.php

@@ -12,6 +12,16 @@ final class PhabricatorSavedQuery extends PhabricatorSearchDAO
       self::CONFIG_SERIALIZATION => array(
         'parameters' => self::SERIALIZATION_JSON,
       ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'engineClassName' => 'text255',
+        'queryKey' => 'bytes12',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_queryKey' => array(
+          'columns' => array('queryKey'),
+          'unique' => true,
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/search/storage/PhabricatorSearchQuery.php

@@ -1,47 +0,0 @@
-<?php
-
-/**
- * Obsolete storage for saved search parameters. This class is no longer used;
- * it was obsoleted by the introduction of {@class:PhabricatorSavedQuery}.
- *
- * This class is retained only because one of the migrations
- * (`20130913.maniphest.1.migratesearch.php`) relies on it to migrate old saved
- * Maniphest searches to new infrastructure. We can remove this class and the
- * corresponding migration after installs have had a reasonable amount of time
- * to perform it.
- *
- * TODO: Remove this class after 2014-09-13, roughly.
- *
- * @deprecated
- */
-final class PhabricatorSearchQuery extends PhabricatorSearchDAO {
-
-  protected $query;
-  protected $parameters = array();
-  protected $queryKey;
-
-  public function getConfiguration() {
-    return array(
-      self::CONFIG_SERIALIZATION => array(
-        'parameters' => self::SERIALIZATION_JSON,
-      ),
-    ) + parent::getConfiguration();
-  }
-
-  public function setParameter($parameter, $value) {
-    $this->parameters[$parameter] = $value;
-    return $this;
-  }
-
-  public function getParameter($parameter, $default = null) {
-    return idx($this->parameters, $parameter, $default);
-  }
-
-  public function save() {
-    if (!$this->getQueryKey()) {
-      $this->setQueryKey(Filesystem::readRandomCharacters(12));
-    }
-    return parent::save();
-  }
-
-}

src/applications/search/storage/PhabricatorSearchSchemaSpec.php

@@ -0,0 +1,9 @@
+<?php
+
+final class PhabricatorSearchSchemaSpec extends PhabricatorConfigSchemaSpec {
+
+  public function buildSchemata() {
+    $this->buildLiskSchemata('PhabricatorSearchDAO');
+  }
+
+}

src/applications/search/storage/document/PhabricatorSearchDocument.php

@@ -2,7 +2,6 @@
 
 final class PhabricatorSearchDocument extends PhabricatorSearchDAO {
 
-  protected $phid;
   protected $documentType;
   protected $documentTitle;
   protected $documentCreated;
@@ -12,6 +11,22 @@ final class PhabricatorSearchDocument extends PhabricatorSearchDAO {
     return array(
       self::CONFIG_TIMESTAMPS => false,
       self::CONFIG_IDS        => self::IDS_MANUAL,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'documentType' => 'text4',
+        'documentTitle' => 'text255',
+        'documentCreated' => 'epoch',
+        'documentModified' => 'epoch',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_phid' => null,
+        'PRIMARY' => array(
+          'columns' => array('phid'),
+          'unique' => true,
+        ),
+        'documentCreated' => array(
+          'columns' => array('documentCreated'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/search/storage/document/PhabricatorSearchDocumentField.php

@@ -2,7 +2,7 @@
 
 final class PhabricatorSearchDocumentField extends PhabricatorSearchDAO {
 
-  protected $phid;
+  protected $phidType;
   protected $field;
   protected $auxPHID;
   protected $corpus;
@@ -11,7 +11,28 @@ final class PhabricatorSearchDocumentField extends PhabricatorSearchDAO {
     return array(
       self::CONFIG_TIMESTAMPS => false,
       self::CONFIG_IDS        => self::IDS_MANUAL,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'phidType' => 'text4',
+        'field' => 'text4',
+        'auxPHID' => 'phid?',
+        'corpus' => 'text?',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_phid' => null,
+        'phid' => array(
+          'columns' => array('phid'),
+        ),
+
+        // NOTE: This is a fulltext index! Be careful!
+        'corpus' => array(
+          'columns' => array('corpus'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 
+  public function getIDKey() {
+    return 'phid';
+  }
+
 }

src/applications/search/storage/document/PhabricatorSearchDocumentRelationship.php

@@ -2,7 +2,6 @@
 
 final class PhabricatorSearchDocumentRelationship extends PhabricatorSearchDAO {
 
-  protected $phid;
   protected $relatedPHID;
   protected $relation;
   protected $relatedType;
@@ -12,7 +11,28 @@ final class PhabricatorSearchDocumentRelationship extends PhabricatorSearchDAO {
     return array(
       self::CONFIG_TIMESTAMPS => false,
       self::CONFIG_IDS        => self::IDS_MANUAL,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'relation' => 'text4',
+        'relatedType' => 'text4',
+        'relatedTime' => 'epoch',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_phid' => null,
+        'phid' => array(
+          'columns' => array('phid'),
+        ),
+        'relatedPHID' => array(
+          'columns' => array('relatedPHID', 'relation'),
+        ),
+        'relation' => array(
+          'columns' => array('relation', 'relatedPHID'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 
+  public function getIDKey() {
+    return 'phid';
+  }
+
 }