From 098d0d93d60c7b6ff235e14d17fd16fe9df283f1 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 1 Oct 2014 07:36:47 -0700 Subject: [PATCH] Generate expected schemata for User/People tables Summary: Ref T1191. Some notes here: - Drops the old LDAP and OAuth info tables. These were migrated to the ExternalAccount table a very long time ago. - Separates surplus/missing keys from other types of surplus/missing things. In the long run, my plan is to have only two notice levels: - Error: something we can't fix (missing database, table, or column; overlong key). - Warning: something we can fix (surplus anything, missing key, bad column type, bad key columns, bad uniqueness, bad collation or charset). - For now, retaining three levels is helpful in generating all the expected scheamta. Test Plan: - Saw ~200 issues resolve, leaving ~1,300. - Grepped for removed tables. Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T1191 Differential Revision: https://secure.phabricator.com/D10580 --- .../20140926.schema.03.dropldapinfo.sql | 1 + .../20140926.schema.04.dropoauthinfo.sql | 1 + src/__phutil_library_map__.php | 2 + .../auth/storage/PhabricatorAuthSession.php | 20 ++++++++++ .../schema/PhabricatorConfigSchemaQuery.php | 15 +++++++- .../schema/PhabricatorConfigSchemaSpec.php | 10 +++++ .../schema/PhabricatorConfigStorageSchema.php | 16 +++++++- .../storage/PhabricatorExternalAccount.php | 25 ++++++++++++ .../people/storage/PhabricatorUser.php | 38 +++++++++++++++++++ .../people/storage/PhabricatorUserEmail.php | 20 ++++++++++ .../people/storage/PhabricatorUserLog.php | 26 +++++++++++++ .../people/storage/PhabricatorUserProfile.php | 16 ++++++++ .../storage/PhabricatorUserSchemaSpec.php | 38 +++++++++++++++++++ .../storage/PhabricatorUserPreferences.php | 6 +++ .../storage/PhabricatorUserSSHKey.php | 21 ++++++++++ .../storage/PhabricatorWorkerActiveTask.php | 24 +++++++++++- .../storage/PhabricatorWorkerArchiveTask.php | 11 ++++++ 17 files changed, 284 insertions(+), 6 deletions(-) create mode 100644 resources/sql/autopatches/20140926.schema.03.dropldapinfo.sql create mode 100644 resources/sql/autopatches/20140926.schema.04.dropoauthinfo.sql create mode 100644 src/applications/people/storage/PhabricatorUserSchemaSpec.php diff --git a/resources/sql/autopatches/20140926.schema.03.dropldapinfo.sql b/resources/sql/autopatches/20140926.schema.03.dropldapinfo.sql new file mode 100644 index 0000000000..11c732d57b --- /dev/null +++ b/resources/sql/autopatches/20140926.schema.03.dropldapinfo.sql @@ -0,0 +1 @@ +DROP TABLE {$NAMESPACE}_user.user_ldapinfo; diff --git a/resources/sql/autopatches/20140926.schema.04.dropoauthinfo.sql b/resources/sql/autopatches/20140926.schema.04.dropoauthinfo.sql new file mode 100644 index 0000000000..fb70d81d00 --- /dev/null +++ b/resources/sql/autopatches/20140926.schema.04.dropoauthinfo.sql @@ -0,0 +1 @@ +DROP TABLE {$NAMESPACE}_user.user_oauthinfo; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 74f9e21459..db99852be3 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2419,6 +2419,7 @@ phutil_register_library_map(array( 'PhabricatorUserRealNameField' => 'applications/people/customfield/PhabricatorUserRealNameField.php', 'PhabricatorUserRolesField' => 'applications/people/customfield/PhabricatorUserRolesField.php', 'PhabricatorUserSSHKey' => 'applications/settings/storage/PhabricatorUserSSHKey.php', + 'PhabricatorUserSchemaSpec' => 'applications/people/storage/PhabricatorUserSchemaSpec.php', 'PhabricatorUserSearchIndexer' => 'applications/people/search/PhabricatorUserSearchIndexer.php', 'PhabricatorUserSinceField' => 'applications/people/customfield/PhabricatorUserSinceField.php', 'PhabricatorUserStatusField' => 'applications/people/customfield/PhabricatorUserStatusField.php', @@ -5417,6 +5418,7 @@ phutil_register_library_map(array( 'PhabricatorUserRealNameField' => 'PhabricatorUserCustomField', 'PhabricatorUserRolesField' => 'PhabricatorUserCustomField', 'PhabricatorUserSSHKey' => 'PhabricatorUserDAO', + 'PhabricatorUserSchemaSpec' => 'PhabricatorConfigSchemaSpec', 'PhabricatorUserSearchIndexer' => 'PhabricatorSearchDocumentIndexer', 'PhabricatorUserSinceField' => 'PhabricatorUserCustomField', 'PhabricatorUserStatusField' => 'PhabricatorUserCustomField', diff --git a/src/applications/auth/storage/PhabricatorAuthSession.php b/src/applications/auth/storage/PhabricatorAuthSession.php index 37ce01e4b5..0c11c3627a 100644 --- a/src/applications/auth/storage/PhabricatorAuthSession.php +++ b/src/applications/auth/storage/PhabricatorAuthSession.php @@ -19,6 +19,26 @@ final class PhabricatorAuthSession extends PhabricatorAuthDAO public function getConfiguration() { return array( self::CONFIG_TIMESTAMPS => false, + self::CONFIG_COLUMN_SCHEMA => array( + 'type' => 'text32', + 'sessionKey' => 'bytes40', + 'sessionStart' => 'epoch', + 'sessionExpires' => 'epoch', + 'highSecurityUntil' => 'epoch?', + 'isPartial' => 'bool', + ), + self::CONFIG_KEY_SCHEMA => array( + 'sessionKey' => array( + 'columns' => array('sessionKey'), + 'unique' => true, + ), + 'key_identity' => array( + 'columns' => array('userPHID', 'type'), + ), + 'key_expires' => array( + 'columns' => array('sessionExpires'), + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php index f1a982fc8c..1a84cf4778 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php @@ -269,12 +269,23 @@ final class PhabricatorConfigSchemaQuery extends Phobject { PhabricatorConfigStorageSchema $expect = null, PhabricatorConfigStorageSchema $actual = null) { + $expect_is_key = ($expect instanceof PhabricatorConfigKeySchema); + $actual_is_key = ($actual instanceof PhabricatorConfigKeySchema); + + if ($expect_is_key || $actual_is_key) { + $missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; + $surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; + } else { + $missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSING; + $surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUS; + } + if (!$expect && !$actual) { throw new Exception(pht('Can not compare two missing schemata!')); } else if ($expect && !$actual) { - $issues = array(PhabricatorConfigStorageSchema::ISSUE_MISSING); + $issues = array($missing_issue); } else if ($actual && !$expect) { - $issues = array(PhabricatorConfigStorageSchema::ISSUE_SURPLUS); + $issues = array($surplus_issue); } else { $issues = $actual->compareTo($expect); } diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index 22fbea4117..52463ad4b8 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -245,6 +245,11 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { $charset = 'binary'; $collation = 'binary'; break; + case 'bytes64': + $column_type = 'char(64)'; + $charset = 'binary'; + $collation = 'binary'; + break; case 'bytes40': $column_type = 'char(40)'; $charset = 'binary'; @@ -278,6 +283,11 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; + case 'text160': + $column_type = 'varchar(160)'; + $charset = $this->getUTF8Charset(); + $collation = $this->getUTF8Collation(); + break; case 'text128': $column_type = 'varchar(128)'; $charset = $this->getUTF8Charset(); diff --git a/src/applications/config/schema/PhabricatorConfigStorageSchema.php b/src/applications/config/schema/PhabricatorConfigStorageSchema.php index c7fe747dd9..9b4d9ee2b9 100644 --- a/src/applications/config/schema/PhabricatorConfigStorageSchema.php +++ b/src/applications/config/schema/PhabricatorConfigStorageSchema.php @@ -3,7 +3,9 @@ abstract class PhabricatorConfigStorageSchema extends Phobject { const ISSUE_MISSING = 'missing'; + const ISSUE_MISSINGKEY = 'missingkey'; const ISSUE_SURPLUS = 'surplus'; + const ISSUE_SURPLUSKEY = 'surpluskey'; const ISSUE_CHARSET = 'charset'; const ISSUE_COLLATION = 'collation'; const ISSUE_COLUMNTYPE = 'columntype'; @@ -102,8 +104,12 @@ abstract class PhabricatorConfigStorageSchema extends Phobject { switch ($issue) { case self::ISSUE_MISSING: return pht('Missing'); + case self::ISSUE_MISSINGKEY: + return pht('Missing Key'); case self::ISSUE_SURPLUS: return pht('Surplus'); + case self::ISSUE_SURPLUSKEY: + return pht('Surplus Key'); case self::ISSUE_CHARSET: return pht('Better Character Set Available'); case self::ISSUE_COLLATION: @@ -131,8 +137,12 @@ abstract class PhabricatorConfigStorageSchema extends Phobject { switch ($issue) { case self::ISSUE_MISSING: return pht('This schema is expected to exist, but does not.'); + case self::ISSUE_MISSINGKEY: + return pht('This key is expected to exist, but does not.'); case self::ISSUE_SURPLUS: return pht('This schema is not expected to exist.'); + case self::ISSUE_SURPLUSKEY: + return pht('This key is not expected to exist.'); case self::ISSUE_CHARSET: return pht('This schema can use a better character set.'); case self::ISSUE_COLLATION: @@ -159,14 +169,16 @@ abstract class PhabricatorConfigStorageSchema extends Phobject { public static function getIssueStatus($issue) { switch ($issue) { case self::ISSUE_MISSING: + case self::ISSUE_MISSINGKEY: case self::ISSUE_SUBFAIL: return self::STATUS_FAIL; case self::ISSUE_SURPLUS: - case self::ISSUE_COLUMNTYPE: + case self::ISSUE_SURPLUSKEY: case self::ISSUE_SUBWARN: + case self::ISSUE_COLUMNTYPE: case self::ISSUE_KEYCOLUMNS: - case self::ISSUE_NULLABLE: case self::ISSUE_UNIQUE: + case self::ISSUE_NULLABLE: return self::STATUS_WARN; case self::ISSUE_SUBNOTE: case self::ISSUE_CHARSET: diff --git a/src/applications/people/storage/PhabricatorExternalAccount.php b/src/applications/people/storage/PhabricatorExternalAccount.php index ba57f24242..6d4a832cf2 100644 --- a/src/applications/people/storage/PhabricatorExternalAccount.php +++ b/src/applications/people/storage/PhabricatorExternalAccount.php @@ -39,6 +39,31 @@ final class PhabricatorExternalAccount extends PhabricatorUserDAO self::CONFIG_SERIALIZATION => array( 'properties' => self::SERIALIZATION_JSON, ), + self::CONFIG_COLUMN_SCHEMA => array( + 'userPHID' => 'phid?', + 'accountType' => 'text16', + 'accountDomain' => 'text64', + 'accountSecret' => 'text?', + 'accountID' => 'text160', + 'displayName' => 'text255?', + 'username' => 'text255?', + 'realName' => 'text255?', + 'email' => 'text255?', + 'emailVerified' => 'bool', + 'profileImagePHID' => 'phid?', + 'accountURI' => 'text255?', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_phid' => null, + 'phid' => array( + 'columns' => array('phid'), + 'unique' => true, + ), + 'account_details' => array( + 'columns' => array('accountType', 'accountDomain', 'accountID'), + 'unique' => true, + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/people/storage/PhabricatorUser.php b/src/applications/people/storage/PhabricatorUser.php index afd33dc4b0..44bf1e05a2 100644 --- a/src/applications/people/storage/PhabricatorUser.php +++ b/src/applications/people/storage/PhabricatorUser.php @@ -113,6 +113,44 @@ final class PhabricatorUser public function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, + self::CONFIG_COLUMN_SCHEMA => array( + 'userName' => 'text64', + 'realName' => 'text128', + 'sex' => 'text4?', + 'translation' => 'text64?', + 'passwordSalt' => 'text32?', + 'passwordHash' => 'text128?', + 'profileImagePHID' => 'phid?', + 'consoleEnabled' => 'bool', + 'consoleVisible' => 'bool', + 'consoleTab' => 'text64', + 'conduitCertificate' => 'text255', + 'isSystemAgent' => 'bool', + 'isDisabled' => 'bool', + 'isAdmin' => 'bool', + 'timezoneIdentifier' => 'text255', + 'isEmailVerified' => 'uint32', + 'isApproved' => 'uint32', + 'accountSecret' => 'bytes64', + 'isEnrolledInMultiFactor' => 'bool', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_phid' => null, + 'phid' => array( + 'columns' => array('phid'), + 'unique' => true, + ), + 'userName' => array( + 'columns' => array('userName'), + 'unique' => true, + ), + 'realName' => array( + 'columns' => array('realName'), + ), + 'key_approved' => array( + 'columns' => array('isApproved'), + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/people/storage/PhabricatorUserEmail.php b/src/applications/people/storage/PhabricatorUserEmail.php index c7b5662b58..e7c2743373 100644 --- a/src/applications/people/storage/PhabricatorUserEmail.php +++ b/src/applications/people/storage/PhabricatorUserEmail.php @@ -14,6 +14,26 @@ final class PhabricatorUserEmail extends PhabricatorUserDAO { const MAX_ADDRESS_LENGTH = 128; + public function getConfiguration() { + return array( + self::CONFIG_COLUMN_SCHEMA => array( + 'address' => 'text128', + 'isVerified' => 'bool', + 'isPrimary' => 'bool', + 'verificationCode' => 'text64?', + ), + self::CONFIG_KEY_SCHEMA => array( + 'address' => array( + 'columns' => array('address'), + 'unique' => true, + ), + 'userPHID' => array( + 'columns' => array('userPHID', 'isPrimary'), + ), + ), + ) + parent::getConfiguration(); + } + public function getVerificationURI() { return '/emailverify/'.$this->getVerificationCode().'/'; } diff --git a/src/applications/people/storage/PhabricatorUserLog.php b/src/applications/people/storage/PhabricatorUserLog.php index 4dcc888880..8a4c2078de 100644 --- a/src/applications/people/storage/PhabricatorUserLog.php +++ b/src/applications/people/storage/PhabricatorUserLog.php @@ -129,6 +129,32 @@ final class PhabricatorUserLog extends PhabricatorUserDAO 'newValue' => self::SERIALIZATION_JSON, 'details' => self::SERIALIZATION_JSON, ), + self::CONFIG_COLUMN_SCHEMA => array( + 'actorPHID' => 'phid?', + 'action' => 'text64', + 'remoteAddr' => 'text64', + 'session' => 'bytes40?', + ), + self::CONFIG_KEY_SCHEMA => array( + 'actorPHID' => array( + 'columns' => array('actorPHID', 'dateCreated'), + ), + 'userPHID' => array( + 'columns' => array('userPHID', 'dateCreated'), + ), + 'action' => array( + 'columns' => array('action', 'dateCreated'), + ), + 'dateCreated' => array( + 'columns' => array('dateCreated'), + ), + 'remoteAddr' => array( + 'columns' => array('remoteAddr', 'dateCreated'), + ), + 'session' => array( + 'columns' => array('session', 'dateCreated'), + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/people/storage/PhabricatorUserProfile.php b/src/applications/people/storage/PhabricatorUserProfile.php index 52f2257856..169bf2d0f3 100644 --- a/src/applications/people/storage/PhabricatorUserProfile.php +++ b/src/applications/people/storage/PhabricatorUserProfile.php @@ -7,4 +7,20 @@ final class PhabricatorUserProfile extends PhabricatorUserDAO { protected $blurb; protected $profileImagePHID; + public function getConfiguration() { + return array( + self::CONFIG_COLUMN_SCHEMA => array( + 'title' => 'text255', + 'blurb' => 'text', + 'profileImagePHID' => 'phid?', + ), + self::CONFIG_KEY_SCHEMA => array( + 'userPHID' => array( + 'columns' => array('userPHID'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + } diff --git a/src/applications/people/storage/PhabricatorUserSchemaSpec.php b/src/applications/people/storage/PhabricatorUserSchemaSpec.php new file mode 100644 index 0000000000..cf484bbc67 --- /dev/null +++ b/src/applications/people/storage/PhabricatorUserSchemaSpec.php @@ -0,0 +1,38 @@ +buildLiskSchemata('PhabricatorUserDAO'); + + $this->buildEdgeSchemata(new PhabricatorUser()); + + $this->buildTransactionSchema( + new PhabricatorUserTransaction()); + + $this->buildCustomFieldSchemata( + new PhabricatorUserConfiguredCustomFieldStorage(), + array( + new PhabricatorUserCustomFieldNumericIndex(), + new PhabricatorUserCustomFieldStringIndex(), + )); + + $this->buildRawSchema( + id(new PhabricatorUser())->getApplicationName(), + PhabricatorUser::NAMETOKEN_TABLE, + array( + 'token' => 'text255', + 'userID' => 'id', + ), + array( + 'token' => array( + 'columns' => array('token'), + ), + 'userID' => array( + 'columns' => array('userID'), + ), + )); + + } + +} diff --git a/src/applications/settings/storage/PhabricatorUserPreferences.php b/src/applications/settings/storage/PhabricatorUserPreferences.php index 39e689352c..233e56c28c 100644 --- a/src/applications/settings/storage/PhabricatorUserPreferences.php +++ b/src/applications/settings/storage/PhabricatorUserPreferences.php @@ -46,6 +46,12 @@ final class PhabricatorUserPreferences extends PhabricatorUserDAO { 'preferences' => self::SERIALIZATION_JSON, ), self::CONFIG_TIMESTAMPS => false, + self::CONFIG_KEY_SCHEMA => array( + 'userPHID' => array( + 'columns' => array('userPHID'), + 'unique' => true, + ), + ), ) + parent::getConfiguration(); } diff --git a/src/applications/settings/storage/PhabricatorUserSSHKey.php b/src/applications/settings/storage/PhabricatorUserSSHKey.php index a06317fedd..2720ecb19f 100644 --- a/src/applications/settings/storage/PhabricatorUserSSHKey.php +++ b/src/applications/settings/storage/PhabricatorUserSSHKey.php @@ -9,6 +9,27 @@ final class PhabricatorUserSSHKey extends PhabricatorUserDAO { protected $keyHash; protected $keyComment; + public function getConfiguration() { + return array( + self::CONFIG_COLUMN_SCHEMA => array( + 'name' => 'text255', + 'keyType' => 'text255', + 'keyBody' => 'text', + 'keyHash' => 'bytes32', + 'keyComment' => 'text255?', + ), + self::CONFIG_KEY_SCHEMA => array( + 'userPHID' => array( + 'columns' => array('userPHID'), + ), + 'keyHash' => array( + 'columns' => array('keyHash'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + public function getEntireKey() { $parts = array( $this->getKeyType(), diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php index be17c4b685..20e3c0d109 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php @@ -8,15 +8,35 @@ final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask { private $localTime; public function getConfiguration() { - return array( + $parent = parent::getConfiguration(); + + $config = array( self::CONFIG_IDS => self::IDS_COUNTER, self::CONFIG_TIMESTAMPS => false, self::CONFIG_KEY_SCHEMA => array( 'dataID' => array( 'columns' => array('dataID'), + 'unique' => true, + ), + 'taskClass' => array( + 'columns' => array('taskClass'), + ), + 'leaseExpires' => array( + 'columns' => array('leaseExpires'), + ), + 'leaseOwner' => array( + 'columns' => array('leaseOwner(16)'), + ), + 'key_failuretime' => array( + 'columns' => array('failureTime'), + ), + 'leaseOwner_2' => array( + 'columns' => array('leaseOwner', 'priority', 'id'), ), ), - ) + parent::getConfiguration(); + ); + + return $config + $parent; } public function setServerTime($server_time) { diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php index 98556e221b..1159708b19 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php @@ -11,10 +11,21 @@ final class PhabricatorWorkerArchiveTask extends PhabricatorWorkerTask { public function getConfiguration() { $config = parent::getConfiguration(); + $config[self::CONFIG_COLUMN_SCHEMA] = array( 'result' => 'uint32', 'duration' => 'uint64', ) + $config[self::CONFIG_COLUMN_SCHEMA]; + + $config[self::CONFIG_KEY_SCHEMA] = array( + 'dateCreated' => array( + 'columns' => array('dateCreated'), + ), + 'leaseOwner' => array( + 'columns' => array('leaseOwner', 'priority', 'id'), + ), + ); + return $config; }