From 84568eba846511e6e0ca1f323712acbbb860fa9d Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 19 Sep 2014 11:46:44 -0700 Subject: [PATCH] Generate expected schemata for Maniphest Summary: Ref T1191. - Adds support for custom fields. - Adds support for partial indexes (indexes on a prefix of a column). - Drops old auxiliary storage table: this was moved to custom field storage about a year ago. - Drops old project table: this was moved to edges about two months ago. Test Plan: - Viewed web UI, saw fewer issues. - Used `grep` to verify no readers/writers for storage or project table. Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T1191 Differential Revision: https://secure.phabricator.com/D10526 --- .../20140919.schema.03.dropaux.sql | 1 + .../20140919.schema.04.droptaskproj.sql | 1 + src/__phutil_library_map__.php | 2 + ...bricatorConfigDatabaseStatusController.php | 10 +++-- .../schema/PhabricatorConfigColumnSchema.php | 33 ++++++++++++-- .../schema/PhabricatorConfigKeySchema.php | 9 ++++ .../schema/PhabricatorConfigSchemaQuery.php | 13 +++++- .../schema/PhabricatorConfigSchemaSpec.php | 18 ++++++++ .../maniphest/storage/ManiphestNameIndex.php | 12 ++++++ .../maniphest/storage/ManiphestSchemaSpec.php | 22 ++++++++++ .../maniphest/storage/ManiphestTask.php | 43 +++++++++++++++++++ .../PhabricatorApplicationTransaction.php | 5 +++ ...abricatorApplicationTransactionComment.php | 1 + ...bricatorCustomFieldNumericIndexStorage.php | 17 ++++++++ .../storage/PhabricatorCustomFieldStorage.php | 10 +++++ ...abricatorCustomFieldStringIndexStorage.php | 17 ++++++++ 16 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 resources/sql/autopatches/20140919.schema.03.dropaux.sql create mode 100644 resources/sql/autopatches/20140919.schema.04.droptaskproj.sql create mode 100644 src/applications/maniphest/storage/ManiphestSchemaSpec.php diff --git a/resources/sql/autopatches/20140919.schema.03.dropaux.sql b/resources/sql/autopatches/20140919.schema.03.dropaux.sql new file mode 100644 index 0000000000..c87dbbdeca --- /dev/null +++ b/resources/sql/autopatches/20140919.schema.03.dropaux.sql @@ -0,0 +1 @@ +DROP TABLE {$NAMESPACE}_maniphest.maniphest_taskauxiliarystorage; diff --git a/resources/sql/autopatches/20140919.schema.04.droptaskproj.sql b/resources/sql/autopatches/20140919.schema.04.droptaskproj.sql new file mode 100644 index 0000000000..1415997b7d --- /dev/null +++ b/resources/sql/autopatches/20140919.schema.04.droptaskproj.sql @@ -0,0 +1 @@ +DROP TABLE {$NAMESPACE}_maniphest.maniphest_taskproject; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 0d767f4508..726f77c881 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -918,6 +918,7 @@ phutil_register_library_map(array( 'ManiphestRemarkupRule' => 'applications/maniphest/remarkup/ManiphestRemarkupRule.php', 'ManiphestReplyHandler' => 'applications/maniphest/mail/ManiphestReplyHandler.php', 'ManiphestReportController' => 'applications/maniphest/controller/ManiphestReportController.php', + 'ManiphestSchemaSpec' => 'applications/maniphest/storage/ManiphestSchemaSpec.php', 'ManiphestSearchIndexer' => 'applications/maniphest/search/ManiphestSearchIndexer.php', 'ManiphestStatusConfigOptionType' => 'applications/maniphest/config/ManiphestStatusConfigOptionType.php', 'ManiphestSubpriorityController' => 'applications/maniphest/controller/ManiphestSubpriorityController.php', @@ -3800,6 +3801,7 @@ phutil_register_library_map(array( 'ManiphestRemarkupRule' => 'PhabricatorObjectRemarkupRule', 'ManiphestReplyHandler' => 'PhabricatorMailReplyHandler', 'ManiphestReportController' => 'ManiphestController', + 'ManiphestSchemaSpec' => 'PhabricatorConfigSchemaSpec', 'ManiphestSearchIndexer' => 'PhabricatorSearchDocumentIndexer', 'ManiphestStatusConfigOptionType' => 'PhabricatorConfigJSONOptionType', 'ManiphestSubpriorityController' => 'ManiphestController', diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php index 1e733222bf..15e41df900 100644 --- a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php +++ b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php @@ -280,6 +280,7 @@ final class PhabricatorConfigDatabaseStatusController $collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION; $nullable_issue = PhabricatorConfigStorageSchema::ISSUE_NULLABLE; $unique_issue = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; + $columns_issue = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; $database = $comp->getDatabase($database_name); if (!$database) { @@ -377,13 +378,14 @@ final class PhabricatorConfigDatabaseStatusController $status = $key->getStatus(); $size = 0; - foreach ($key->getColumnNames() as $column_name) { + foreach ($key->getColumnNames() as $column_spec) { + list($column_name, $prefix) = $key->getKeyColumnAndPrefix($column_spec); $column = $table->getColumn($column_name); if (!$column) { $size = 0; break; } - $size += $column->getKeyByteLength(); + $size += $column->getKeyByteLength($prefix); } $size_formatted = null; @@ -406,7 +408,9 @@ final class PhabricatorConfigDatabaseStatusController $key_name.'/'), ), $key_name), - implode(', ', $key->getColumnNames()), + $this->renderAttr( + implode(', ', $key->getColumnNames()), + $key->hasIssue($columns_issue)), $this->renderAttr( $this->renderBoolean($key->getUnique()), $key->hasIssue($unique_issue)), diff --git a/src/applications/config/schema/PhabricatorConfigColumnSchema.php b/src/applications/config/schema/PhabricatorConfigColumnSchema.php index 7bad159afa..900ba1945f 100644 --- a/src/applications/config/schema/PhabricatorConfigColumnSchema.php +++ b/src/applications/config/schema/PhabricatorConfigColumnSchema.php @@ -58,20 +58,47 @@ final class PhabricatorConfigColumnSchema return $this->characterSet; } - public function getKeyByteLength() { + public function getKeyByteLength($prefix = null) { $type = $this->getColumnType(); $matches = null; if (preg_match('/^varchar\((\d+)\)$/', $type, $matches)) { // For utf8mb4, each character requires 4 bytes. - return ((int)$matches[1]) * 4; + $size = (int)$matches[1]; + if ($prefix && $prefix < $size) { + $size = $prefix; + } + return $size * 4; } $matches = null; if (preg_match('/^char\((\d+)\)$/', $type, $matches)) { // We use char() only for fixed-length binary data, so its size // is always the column size. - return ((int)$matches[1]); + $size = (int)$matches[1]; + if ($prefix && $prefix < $size) { + $size = $prefix; + } + return $size; + } + + // The "long..." types are arbitrarily long, so just use a big number to + // get the point across. In practice, these should always index only a + // prefix. + if ($type == 'longtext') { + $size = (1 << 16); + if ($prefix && $prefix < $size) { + $size = $prefix; + } + return $size * 4; + } + + if ($type == 'longblob') { + $size = (1 << 16); + if ($prefix && $prefix < $size) { + $size = $prefix; + } + return $size * 1; } switch ($type) { diff --git a/src/applications/config/schema/PhabricatorConfigKeySchema.php b/src/applications/config/schema/PhabricatorConfigKeySchema.php index 70aad32ee2..1e2cb4c654 100644 --- a/src/applications/config/schema/PhabricatorConfigKeySchema.php +++ b/src/applications/config/schema/PhabricatorConfigKeySchema.php @@ -28,6 +28,15 @@ final class PhabricatorConfigKeySchema return array(); } + public function getKeyColumnAndPrefix($column_name) { + $matches = null; + if (preg_match('/^(.*)\((\d+)\)\z/', $column_name, $matches)) { + return array($matches[1], (int)$matches[2]); + } else { + return array($column_name, null); + } + } + public function compareToSimilarSchema( PhabricatorConfigStorageSchema $expect) { diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php index b5c9593b72..f1a982fc8c 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php @@ -115,9 +115,20 @@ final class PhabricatorConfigSchemaQuery extends Phobject { foreach ($keys as $key_name => $key_pieces) { $key_pieces = isort($key_pieces, 'Seq_in_index'); $head = head($key_pieces); + + // This handles string indexes which index only a prefix of a field. + $column_names = array(); + foreach ($key_pieces as $piece) { + $name = $piece['Column_name']; + if ($piece['Sub_part']) { + $name = $name.'('.$piece['Sub_part'].')'; + } + $column_names[] = $name; + } + $key_schema = id(new PhabricatorConfigKeySchema()) ->setName($key_name) - ->setColumnNames(ipull($key_pieces, 'Column_name')) + ->setColumnNames($column_names) ->setUnique(!$head['Non_unique']); $table_schema->addKey($key_schema); diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index 555f373ec2..eda715639d 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -56,6 +56,16 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { } } + protected function buildCustomFieldSchemata( + PhabricatorLiskDAO $storage, + array $indexes) { + + $this->buildLiskObjectSchema($storage); + foreach ($indexes as $index) { + $this->buildLiskObjectSchema($index); + } + } + private function buildLiskObjectSchema(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), @@ -123,6 +133,10 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { array( 'PRIMARY' => array( 'columns' => array('src', 'type', 'dst'), + 'unique' => true, + ), + 'src' => array( + 'columns' => array('src', 'type', 'dateCreated', 'seq'), ), )); @@ -136,6 +150,7 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { array( 'PRIMARY' => array( 'columns' => array('id'), + 'unique' => true, ), )); } @@ -217,6 +232,9 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { case 'uint64': $column_type = 'bigint(20) unsigned'; break; + case 'sint64': + $column_type = 'bigint(20)'; + break; case 'phid': case 'policy'; $column_type = 'varchar(64)'; diff --git a/src/applications/maniphest/storage/ManiphestNameIndex.php b/src/applications/maniphest/storage/ManiphestNameIndex.php index 196dcad9f6..7988dc90f9 100644 --- a/src/applications/maniphest/storage/ManiphestNameIndex.php +++ b/src/applications/maniphest/storage/ManiphestNameIndex.php @@ -12,6 +12,18 @@ final class ManiphestNameIndex extends ManiphestDAO { public function getConfiguration() { return array( self::CONFIG_TIMESTAMPS => false, + self::CONFIG_COLUMN_SCHEMA => array( + 'indexedObjectName' => 'text128', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_phid' => array( + 'columns' => array('indexedObjectPHID'), + 'unique' => true, + ), + 'key_name' => array( + 'columns' => array('indexedObjectName'), + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/maniphest/storage/ManiphestSchemaSpec.php b/src/applications/maniphest/storage/ManiphestSchemaSpec.php new file mode 100644 index 0000000000..72d34934b9 --- /dev/null +++ b/src/applications/maniphest/storage/ManiphestSchemaSpec.php @@ -0,0 +1,22 @@ +buildLiskSchemata('ManiphestDAO'); + + $this->buildEdgeSchemata(new ManiphestTask()); + $this->buildTransactionSchema( + new ManiphestTransaction(), + new ManiphestTransactionComment()); + + $this->buildCustomFieldSchemata( + new ManiphestCustomFieldStorage(), + array( + new ManiphestCustomFieldNumericIndex(), + new ManiphestCustomFieldStringIndex(), + )); + } + +} diff --git a/src/applications/maniphest/storage/ManiphestTask.php b/src/applications/maniphest/storage/ManiphestTask.php index 2a988f95c4..1174bd68ff 100644 --- a/src/applications/maniphest/storage/ManiphestTask.php +++ b/src/applications/maniphest/storage/ManiphestTask.php @@ -67,6 +67,49 @@ final class ManiphestTask extends ManiphestDAO 'attached' => self::SERIALIZATION_JSON, 'projectPHIDs' => self::SERIALIZATION_JSON, ), + self::CONFIG_COLUMN_SCHEMA => array( + 'ownerPHID' => 'phid?', + 'status' => 'text12', + 'priority' => 'uint32', + 'title' => 'text', + 'originalTitle' => 'text', + 'description' => 'text', + 'mailKey' => 'bytes20', + 'ownerOrdering' => 'text64?', + 'originalEmailSource' => 'text255?', + 'subpriority' => 'double', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_phid' => null, + 'phid' => array( + 'columns' => array('phid'), + 'unique' => true, + ), + 'priority' => array( + 'columns' => array('priority', 'status'), + ), + 'status' => array( + 'columns' => array('status'), + ), + 'ownerPHID' => array( + 'columns' => array('ownerPHID', 'status'), + ), + 'authorPHID' => array( + 'columns' => array('authorPHID', 'status'), + ), + 'ownerOrdering' => array( + 'columns' => array('ownerOrdering'), + ), + 'priority_2' => array( + 'columns' => array('priority', 'subpriority'), + ), + 'key_dateCreated' => array( + 'columns' => array('dateCreated'), + ), + 'key_dateModified' => array( + 'columns' => array('dateModified'), + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php index 3c4b124655..da9561ac71 100644 --- a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php +++ b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php @@ -113,6 +113,11 @@ abstract class PhabricatorApplicationTransaction 'contentSource' => 'text', 'transactionType' => 'text32', ), + self::CONFIG_KEY_SCHEMA => array( + 'key_object' => array( + 'columns' => array('objectPHID'), + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/transactions/storage/PhabricatorApplicationTransactionComment.php b/src/applications/transactions/storage/PhabricatorApplicationTransactionComment.php index 0beb04f12e..cacd577f7a 100644 --- a/src/applications/transactions/storage/PhabricatorApplicationTransactionComment.php +++ b/src/applications/transactions/storage/PhabricatorApplicationTransactionComment.php @@ -38,6 +38,7 @@ abstract class PhabricatorApplicationTransactionComment self::CONFIG_KEY_SCHEMA => array( 'key_version' => array( 'columns' => array('transactionPHID', 'commentVersion'), + 'unique' => true, ), ), ) + parent::getConfiguration(); diff --git a/src/infrastructure/customfield/storage/PhabricatorCustomFieldNumericIndexStorage.php b/src/infrastructure/customfield/storage/PhabricatorCustomFieldNumericIndexStorage.php index 0b98bad40a..2dc22cca10 100644 --- a/src/infrastructure/customfield/storage/PhabricatorCustomFieldNumericIndexStorage.php +++ b/src/infrastructure/customfield/storage/PhabricatorCustomFieldNumericIndexStorage.php @@ -3,6 +3,23 @@ abstract class PhabricatorCustomFieldNumericIndexStorage extends PhabricatorCustomFieldIndexStorage { + public function getConfiguration() { + return array( + self::CONFIG_COLUMN_SCHEMA => array( + 'indexKey' => 'bytes12', + 'indexValue' => 'sint64', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_join' => array( + 'columns' => array('objectPHID', 'indexKey', 'indexValue'), + ), + 'key_find' => array( + 'columns' => array('indexKey', 'indexValue'), + ), + ), + ) + parent::getConfiguration(); + } + public function formatForInsert(AphrontDatabaseConnection $conn) { return qsprintf( $conn, diff --git a/src/infrastructure/customfield/storage/PhabricatorCustomFieldStorage.php b/src/infrastructure/customfield/storage/PhabricatorCustomFieldStorage.php index 31e3c25165..3c586e81b5 100644 --- a/src/infrastructure/customfield/storage/PhabricatorCustomFieldStorage.php +++ b/src/infrastructure/customfield/storage/PhabricatorCustomFieldStorage.php @@ -10,6 +10,16 @@ abstract class PhabricatorCustomFieldStorage public function getConfiguration() { return array( self::CONFIG_TIMESTAMPS => false, + self::CONFIG_COLUMN_SCHEMA => array( + 'fieldIndex' => 'bytes12', + 'fieldValue' => 'text', + ), + self::CONFIG_KEY_SCHEMA => array( + 'objectPHID' => array( + 'columns' => array('objectPHID', 'fieldIndex'), + 'unique' => true, + ), + ), ) + parent::getConfiguration(); } diff --git a/src/infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php b/src/infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php index 1cb840a89f..51d5449dc8 100644 --- a/src/infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php +++ b/src/infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php @@ -3,6 +3,23 @@ abstract class PhabricatorCustomFieldStringIndexStorage extends PhabricatorCustomFieldIndexStorage { + public function getConfiguration() { + return array( + self::CONFIG_COLUMN_SCHEMA => array( + 'indexKey' => 'bytes12', + 'indexValue' => 'text', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_join' => array( + 'columns' => array('objectPHID', 'indexKey', 'indexValue(64)'), + ), + 'key_find' => array( + 'columns' => array('indexKey', 'indexValue(64)'), + ), + ), + ) + parent::getConfiguration(); + } + public function formatForInsert(AphrontDatabaseConnection $conn) { return qsprintf( $conn,