Use binary collations for most text

Summary:
Ref T1191. For most text columns, we either don't care if "a" and "A" are the same, or we expect them to be different (for example: keys, domains, secrets, etc). Default text columns to the `_bin` collation so they are compared by strict character value. This is safer in cases where we aren't sure.

For some text columns, we allow the user to sort by the column in the UI (like Maniphest task titles) or we do care that "A" and "a" are the same (for example: project names). Introduce a new class of virtual data types, the "sort..." types, to cover these columns. These are like the "text..." types but use sorting collations which treat "A" and "a" the same.

Test Plan:
  - Made an effort to identify all columns where the UI relies on database collation.
  - Ran `bin/storage adjust` and cleared all warnings.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: beng, epriestley

Maniphest Tasks: T1191

Differential Revision: https://secure.phabricator.com/D10602

src/applications/config/schema/PhabricatorConfigSchemaQuery.php

@@ -154,7 +154,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
       // collation. This is most correct, and will sort properly.
 
       $utf8_charset = 'utf8mb4';
-      $utf8_collation = 'utf8mb4_unicode_ci';
+      $utf8_binary_collation = 'utf8mb4_bin';
+      $utf8_sorting_collation = 'utf8mb4_unicode_ci';
     } else {
       // If utf8mb4 is not available, we use binary. This allows us to store
       // 4-byte unicode characters. This has some tradeoffs:
@@ -167,7 +168,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
       // to prevent this.
 
       $utf8_charset = 'binary';
-      $utf8_collation = 'binary';
+      $utf8_binary_collation = 'binary';
+      $utf8_sorting_collation = 'binary';
     }
 
     $specs = id(new PhutilSymbolLoader())
@@ -177,8 +179,9 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
     $server_schema = new PhabricatorConfigServerSchema();
     foreach ($specs as $spec) {
       $spec
-        ->setUTF8Collation($utf8_collation)
         ->setUTF8Charset($utf8_charset)
+        ->setUTF8BinaryCollation($utf8_binary_collation)
+        ->setUTF8SortingCollation($utf8_sorting_collation)
         ->setServer($server_schema)
         ->buildSchemata($server_schema);
     }

src/applications/config/schema/PhabricatorConfigSchemaSpec.php

@@ -4,15 +4,25 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
 
   private $server;
   private $utf8Charset;
-  private $utf8Collation;
+  private $utf8BinaryCollation;
+  private $utf8SortingCollation;
 
-  public function setUTF8Collation($utf8_collation) {
+  public function setUTF8SortingCollation($utf8_sorting_collation) {
-    $this->utf8Collation = $utf8_collation;
+    $this->utf8SortingCollation = $utf8_sorting_collation;
     return $this;
   }
 
-  public function getUTF8Collation() {
+  public function getUTF8SortingCollation() {
-    return $this->utf8Collation;
+    return $this->utf8SortingCollation;
+  }
+
+  public function setUTF8BinaryCollation($utf8_binary_collation) {
+    $this->utf8BinaryCollation = $utf8_binary_collation;
+    return $this;
+  }
+
+  public function getUTF8BinaryCollation() {
+    return $this->utf8BinaryCollation;
   }
 
   public function setUTF8Charset($utf8_charset) {
@@ -195,7 +205,7 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
     return id(new PhabricatorConfigDatabaseSchema())
       ->setName($this->getNamespacedDatabase($name))
       ->setCharacterSet($this->getUTF8Charset())
-      ->setCollation($this->getUTF8Collation());
+      ->setCollation($this->getUTF8BinaryCollation());
   }
 
   protected function getNamespacedDatabase($name) {
@@ -206,7 +216,7 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
   protected function newTable($name) {
     return id(new PhabricatorConfigTableSchema())
       ->setName($name)
-      ->setCollation($this->getUTF8Collation());
+      ->setCollation($this->getUTF8BinaryCollation());
   }
 
   protected function newColumn($name) {
@@ -276,70 +286,95 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
       case 'bytes':
         $column_type = 'longblob';
         break;
+      case 'sort255':
+        $column_type = 'varchar(255)';
+        $charset = $this->getUTF8Charset();
+        $collation = $this->getUTF8SortingCollation();
+        break;
+      case 'sort128':
+        $column_type = 'varchar(128)';
+        $charset = $this->getUTF8Charset();
+        $collation = $this->getUTF8SortingCollation();
+        break;
+      case 'sort64':
+        $column_type = 'varchar(64)';
+        $charset = $this->getUTF8Charset();
+        $collation = $this->getUTF8SortingCollation();
+        break;
+      case 'sort32':
+        $column_type = 'varchar(32)';
+        $charset = $this->getUTF8Charset();
+        $collation = $this->getUTF8SortingCollation();
+        break;
+      case 'sort':
+        $column_type = 'longtext';
+        $charset = $this->getUTF8Charset();
+        $collation = $this->getUTF8SortingCollation();
+        break;
       case 'text255':
         $column_type = 'varchar(255)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text160':
         $column_type = 'varchar(160)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text128':
         $column_type = 'varchar(128)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text80':
         $column_type = 'varchar(80)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text64':
         $column_type = 'varchar(64)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text40':
         $column_type = 'varchar(40)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text32':
         $column_type = 'varchar(32)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text20':
         $column_type = 'varchar(20)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text16':
         $column_type = 'varchar(16)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text12':
         $column_type = 'varchar(12)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text8':
         $column_type = 'varchar(8)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text4':
         $column_type = 'varchar(4)';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'text':
         $column_type = 'longtext';
         $charset = $this->getUTF8Charset();
-        $collation = $this->getUTF8Collation();
+        $collation = $this->getUTF8BinaryCollation();
         break;
       case 'bool':
         $column_type = 'tinyint(1)';

src/applications/maniphest/storage/ManiphestNameIndex.php

@@ -13,7 +13,7 @@ final class ManiphestNameIndex extends ManiphestDAO {
     return array(
       self::CONFIG_TIMESTAMPS => false,
       self::CONFIG_COLUMN_SCHEMA => array(
-        'indexedObjectName' => 'text128',
+        'indexedObjectName' => 'sort128',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_phid' => array(

src/applications/maniphest/storage/ManiphestTask.php

@@ -71,7 +71,7 @@ final class ManiphestTask extends ManiphestDAO
         'ownerPHID' => 'phid?',
         'status' => 'text12',
         'priority' => 'uint32',
-        'title' => 'text',
+        'title' => 'sort',
         'originalTitle' => 'text',
         'description' => 'text',
         'mailKey' => 'bytes20',
@@ -114,6 +114,9 @@ final class ManiphestTask extends ManiphestDAO
         'key_dateModified' => array(
           'columns' => array('dateModified'),
         ),
+        'key_title' => array(
+          'columns' => array('title(64)'),
+        ),
       ),
     ) + parent::getConfiguration();
   }

src/applications/phame/storage/PhamePost.php

@@ -90,7 +90,7 @@ final class PhamePost extends PhameDAO
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
         'title' => 'text255',
-        'phameTitle' => 'text64',
+        'phameTitle' => 'sort64',
         'visibility' => 'uint32',
 
         // T6203/NULLABILITY

src/applications/phriction/storage/PhrictionContent.php

@@ -32,7 +32,7 @@ final class PhrictionContent extends PhrictionDAO
     return array(
       self::CONFIG_COLUMN_SCHEMA => array(
         'version' => 'uint32',
-        'title' => 'text',
+        'title' => 'sort',
         'slug' => 'text128',
         'content' => 'text',
         'changeType' => 'uint32',

src/applications/phriction/storage/PhrictionDocument.php

@@ -25,7 +25,7 @@ final class PhrictionDocument extends PhrictionDAO
       self::CONFIG_AUX_PHID   => true,
       self::CONFIG_TIMESTAMPS => false,
       self::CONFIG_COLUMN_SCHEMA => array(
-        'slug' => 'text128',
+        'slug' => 'sort128',
         'depth' => 'uint32',
         'contentID' => 'id?',
         'status' => 'uint32',

src/applications/project/storage/PhabricatorProject.php

@@ -122,7 +122,7 @@ final class PhabricatorProject extends PhabricatorProjectDAO
         'subprojectPHIDs' => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text128',
+        'name' => 'sort128',
         'status' => 'text32',
         'phrictionSlug' => 'text128?',
         'isMembershipLocked' => 'bool',

src/applications/repository/storage/PhabricatorRepository.php

@@ -79,8 +79,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
         'details' => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'sort255',
-        'callsign' => 'text32',
+        'callsign' => 'sort32',
         'versionControlSystem' => 'text32',
         'uuid' => 'text64?',
         'pushPolicy' => 'policy',

src/infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php

@@ -7,7 +7,7 @@ abstract class PhabricatorCustomFieldStringIndexStorage
     return array(
       self::CONFIG_COLUMN_SCHEMA => array(
         'indexKey' => 'bytes12',
-        'indexValue' => 'text',
+        'indexValue' => 'sort',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_join' => array(