mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-22 14:52:41 +01:00
Use binary collations for most text
Summary: Ref T1191. For most text columns, we either don't care if "a" and "A" are the same, or we expect them to be different (for example: keys, domains, secrets, etc). Default text columns to the `_bin` collation so they are compared by strict character value. This is safer in cases where we aren't sure. For some text columns, we allow the user to sort by the column in the UI (like Maniphest task titles) or we do care that "A" and "a" are the same (for example: project names). Introduce a new class of virtual data types, the "sort..." types, to cover these columns. These are like the "text..." types but use sorting collations which treat "A" and "a" the same. Test Plan: - Made an effort to identify all columns where the UI relies on database collation. - Ran `bin/storage adjust` and cleared all warnings. Reviewers: btrahan Reviewed By: btrahan Subscribers: beng, epriestley Maniphest Tasks: T1191 Differential Revision: https://secure.phabricator.com/D10602
This commit is contained in:
parent
4fcc634a99
commit
1dfa94e571
10 changed files with 73 additions and 32 deletions
|
@ -154,7 +154,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
|
|||
// collation. This is most correct, and will sort properly.
|
||||
|
||||
$utf8_charset = 'utf8mb4';
|
||||
$utf8_collation = 'utf8mb4_unicode_ci';
|
||||
$utf8_binary_collation = 'utf8mb4_bin';
|
||||
$utf8_sorting_collation = 'utf8mb4_unicode_ci';
|
||||
} else {
|
||||
// If utf8mb4 is not available, we use binary. This allows us to store
|
||||
// 4-byte unicode characters. This has some tradeoffs:
|
||||
|
@ -167,7 +168,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
|
|||
// to prevent this.
|
||||
|
||||
$utf8_charset = 'binary';
|
||||
$utf8_collation = 'binary';
|
||||
$utf8_binary_collation = 'binary';
|
||||
$utf8_sorting_collation = 'binary';
|
||||
}
|
||||
|
||||
$specs = id(new PhutilSymbolLoader())
|
||||
|
@ -177,8 +179,9 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
|
|||
$server_schema = new PhabricatorConfigServerSchema();
|
||||
foreach ($specs as $spec) {
|
||||
$spec
|
||||
->setUTF8Collation($utf8_collation)
|
||||
->setUTF8Charset($utf8_charset)
|
||||
->setUTF8BinaryCollation($utf8_binary_collation)
|
||||
->setUTF8SortingCollation($utf8_sorting_collation)
|
||||
->setServer($server_schema)
|
||||
->buildSchemata($server_schema);
|
||||
}
|
||||
|
|
|
@ -4,15 +4,25 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
|
|||
|
||||
private $server;
|
||||
private $utf8Charset;
|
||||
private $utf8Collation;
|
||||
private $utf8BinaryCollation;
|
||||
private $utf8SortingCollation;
|
||||
|
||||
public function setUTF8Collation($utf8_collation) {
|
||||
$this->utf8Collation = $utf8_collation;
|
||||
public function setUTF8SortingCollation($utf8_sorting_collation) {
|
||||
$this->utf8SortingCollation = $utf8_sorting_collation;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getUTF8Collation() {
|
||||
return $this->utf8Collation;
|
||||
public function getUTF8SortingCollation() {
|
||||
return $this->utf8SortingCollation;
|
||||
}
|
||||
|
||||
public function setUTF8BinaryCollation($utf8_binary_collation) {
|
||||
$this->utf8BinaryCollation = $utf8_binary_collation;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getUTF8BinaryCollation() {
|
||||
return $this->utf8BinaryCollation;
|
||||
}
|
||||
|
||||
public function setUTF8Charset($utf8_charset) {
|
||||
|
@ -195,7 +205,7 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
|
|||
return id(new PhabricatorConfigDatabaseSchema())
|
||||
->setName($this->getNamespacedDatabase($name))
|
||||
->setCharacterSet($this->getUTF8Charset())
|
||||
->setCollation($this->getUTF8Collation());
|
||||
->setCollation($this->getUTF8BinaryCollation());
|
||||
}
|
||||
|
||||
protected function getNamespacedDatabase($name) {
|
||||
|
@ -206,7 +216,7 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
|
|||
protected function newTable($name) {
|
||||
return id(new PhabricatorConfigTableSchema())
|
||||
->setName($name)
|
||||
->setCollation($this->getUTF8Collation());
|
||||
->setCollation($this->getUTF8BinaryCollation());
|
||||
}
|
||||
|
||||
protected function newColumn($name) {
|
||||
|
@ -276,70 +286,95 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
|
|||
case 'bytes':
|
||||
$column_type = 'longblob';
|
||||
break;
|
||||
case 'sort255':
|
||||
$column_type = 'varchar(255)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8SortingCollation();
|
||||
break;
|
||||
case 'sort128':
|
||||
$column_type = 'varchar(128)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8SortingCollation();
|
||||
break;
|
||||
case 'sort64':
|
||||
$column_type = 'varchar(64)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8SortingCollation();
|
||||
break;
|
||||
case 'sort32':
|
||||
$column_type = 'varchar(32)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8SortingCollation();
|
||||
break;
|
||||
case 'sort':
|
||||
$column_type = 'longtext';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8SortingCollation();
|
||||
break;
|
||||
case 'text255':
|
||||
$column_type = 'varchar(255)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text160':
|
||||
$column_type = 'varchar(160)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text128':
|
||||
$column_type = 'varchar(128)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text80':
|
||||
$column_type = 'varchar(80)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text64':
|
||||
$column_type = 'varchar(64)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text40':
|
||||
$column_type = 'varchar(40)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text32':
|
||||
$column_type = 'varchar(32)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text20':
|
||||
$column_type = 'varchar(20)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text16':
|
||||
$column_type = 'varchar(16)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text12':
|
||||
$column_type = 'varchar(12)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text8':
|
||||
$column_type = 'varchar(8)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text4':
|
||||
$column_type = 'varchar(4)';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'text':
|
||||
$column_type = 'longtext';
|
||||
$charset = $this->getUTF8Charset();
|
||||
$collation = $this->getUTF8Collation();
|
||||
$collation = $this->getUTF8BinaryCollation();
|
||||
break;
|
||||
case 'bool':
|
||||
$column_type = 'tinyint(1)';
|
||||
|
|
|
@ -13,7 +13,7 @@ final class ManiphestNameIndex extends ManiphestDAO {
|
|||
return array(
|
||||
self::CONFIG_TIMESTAMPS => false,
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'indexedObjectName' => 'text128',
|
||||
'indexedObjectName' => 'sort128',
|
||||
),
|
||||
self::CONFIG_KEY_SCHEMA => array(
|
||||
'key_phid' => array(
|
||||
|
|
|
@ -71,7 +71,7 @@ final class ManiphestTask extends ManiphestDAO
|
|||
'ownerPHID' => 'phid?',
|
||||
'status' => 'text12',
|
||||
'priority' => 'uint32',
|
||||
'title' => 'text',
|
||||
'title' => 'sort',
|
||||
'originalTitle' => 'text',
|
||||
'description' => 'text',
|
||||
'mailKey' => 'bytes20',
|
||||
|
@ -114,6 +114,9 @@ final class ManiphestTask extends ManiphestDAO
|
|||
'key_dateModified' => array(
|
||||
'columns' => array('dateModified'),
|
||||
),
|
||||
'key_title' => array(
|
||||
'columns' => array('title(64)'),
|
||||
),
|
||||
),
|
||||
) + parent::getConfiguration();
|
||||
}
|
||||
|
|
|
@ -90,7 +90,7 @@ final class PhamePost extends PhameDAO
|
|||
),
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'title' => 'text255',
|
||||
'phameTitle' => 'text64',
|
||||
'phameTitle' => 'sort64',
|
||||
'visibility' => 'uint32',
|
||||
|
||||
// T6203/NULLABILITY
|
||||
|
|
|
@ -32,7 +32,7 @@ final class PhrictionContent extends PhrictionDAO
|
|||
return array(
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'version' => 'uint32',
|
||||
'title' => 'text',
|
||||
'title' => 'sort',
|
||||
'slug' => 'text128',
|
||||
'content' => 'text',
|
||||
'changeType' => 'uint32',
|
||||
|
|
|
@ -25,7 +25,7 @@ final class PhrictionDocument extends PhrictionDAO
|
|||
self::CONFIG_AUX_PHID => true,
|
||||
self::CONFIG_TIMESTAMPS => false,
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'slug' => 'text128',
|
||||
'slug' => 'sort128',
|
||||
'depth' => 'uint32',
|
||||
'contentID' => 'id?',
|
||||
'status' => 'uint32',
|
||||
|
|
|
@ -122,7 +122,7 @@ final class PhabricatorProject extends PhabricatorProjectDAO
|
|||
'subprojectPHIDs' => self::SERIALIZATION_JSON,
|
||||
),
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'name' => 'text128',
|
||||
'name' => 'sort128',
|
||||
'status' => 'text32',
|
||||
'phrictionSlug' => 'text128?',
|
||||
'isMembershipLocked' => 'bool',
|
||||
|
|
|
@ -79,8 +79,8 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
|
|||
'details' => self::SERIALIZATION_JSON,
|
||||
),
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'name' => 'text255',
|
||||
'callsign' => 'text32',
|
||||
'name' => 'sort255',
|
||||
'callsign' => 'sort32',
|
||||
'versionControlSystem' => 'text32',
|
||||
'uuid' => 'text64?',
|
||||
'pushPolicy' => 'policy',
|
||||
|
|
|
@ -7,7 +7,7 @@ abstract class PhabricatorCustomFieldStringIndexStorage
|
|||
return array(
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'indexKey' => 'bytes12',
|
||||
'indexValue' => 'text',
|
||||
'indexValue' => 'sort',
|
||||
),
|
||||
self::CONFIG_KEY_SCHEMA => array(
|
||||
'key_join' => array(
|
||||
|
|
Loading…
Reference in a new issue