Generate expected schemata for Doorkeeper, Draft, Drydock, Feed

Summary:
Ref T1191. Notable:

  - Allowed objects to remove default columns (some feed tables have no `id`).
  - Added a "note" severity and moved all the charset stuff down to that to make progress more clear.

Test Plan:
Trying to make the whole thing blue...

{F205970}

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T1191

Differential Revision: https://secure.phabricator.com/D10519

src/__phutil_library_map__.php

@@ -570,6 +570,7 @@ phutil_register_library_map(array(
     'DoorkeeperRemarkupRule' => 'applications/doorkeeper/remarkup/DoorkeeperRemarkupRule.php',
     'DoorkeeperRemarkupRuleAsana' => 'applications/doorkeeper/remarkup/DoorkeeperRemarkupRuleAsana.php',
     'DoorkeeperRemarkupRuleJIRA' => 'applications/doorkeeper/remarkup/DoorkeeperRemarkupRuleJIRA.php',
+    'DoorkeeperSchemaSpec' => 'applications/doorkeeper/storage/DoorkeeperSchemaSpec.php',
     'DoorkeeperTagView' => 'applications/doorkeeper/view/DoorkeeperTagView.php',
     'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php',
     'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php',
@@ -636,6 +637,7 @@ phutil_register_library_map(array(
     'DrydockResourceViewController' => 'applications/drydock/controller/DrydockResourceViewController.php',
     'DrydockSFTPFilesystemInterface' => 'applications/drydock/interface/filesystem/DrydockSFTPFilesystemInterface.php',
     'DrydockSSHCommandInterface' => 'applications/drydock/interface/command/DrydockSSHCommandInterface.php',
+    'DrydockSchemaSpec' => 'applications/drydock/storage/DrydockSchemaSpec.php',
     'DrydockWebrootInterface' => 'applications/drydock/interface/webroot/DrydockWebrootInterface.php',
     'DrydockWorkingCopyBlueprintImplementation' => 'applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php',
     'FeedConduitAPIMethod' => 'applications/feed/conduit/FeedConduitAPIMethod.php',
@@ -1514,6 +1516,7 @@ phutil_register_library_map(array(
     'PhabricatorDoorkeeperApplication' => 'applications/doorkeeper/application/PhabricatorDoorkeeperApplication.php',
     'PhabricatorDraft' => 'applications/draft/storage/PhabricatorDraft.php',
     'PhabricatorDraftDAO' => 'applications/draft/storage/PhabricatorDraftDAO.php',
+    'PhabricatorDraftSchemaSpec' => 'applications/draft/storage/PhabricatorDraftSchemaSpec.php',
     'PhabricatorDrydockApplication' => 'applications/drydock/application/PhabricatorDrydockApplication.php',
     'PhabricatorEdgeConfig' => 'infrastructure/edges/constants/PhabricatorEdgeConfig.php',
     'PhabricatorEdgeConstants' => 'infrastructure/edges/constants/PhabricatorEdgeConstants.php',
@@ -1574,6 +1577,7 @@ phutil_register_library_map(array(
     'PhabricatorFeedManagementWorkflow' => 'applications/feed/management/PhabricatorFeedManagementWorkflow.php',
     'PhabricatorFeedPublicStreamController' => 'applications/feed/controller/PhabricatorFeedPublicStreamController.php',
     'PhabricatorFeedQuery' => 'applications/feed/query/PhabricatorFeedQuery.php',
+    'PhabricatorFeedSchemaSpec' => 'applications/feed/storage/PhabricatorFeedSchemaSpec.php',
     'PhabricatorFeedSearchEngine' => 'applications/feed/query/PhabricatorFeedSearchEngine.php',
     'PhabricatorFeedStory' => 'applications/feed/story/PhabricatorFeedStory.php',
     'PhabricatorFeedStoryAggregate' => 'applications/feed/story/PhabricatorFeedStoryAggregate.php',
@@ -3373,6 +3377,7 @@ phutil_register_library_map(array(
     'DoorkeeperRemarkupRule' => 'PhutilRemarkupRule',
     'DoorkeeperRemarkupRuleAsana' => 'DoorkeeperRemarkupRule',
     'DoorkeeperRemarkupRuleJIRA' => 'DoorkeeperRemarkupRule',
+    'DoorkeeperSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'DoorkeeperTagView' => 'AphrontView',
     'DoorkeeperTagsController' => 'PhabricatorController',
     'DrydockAllocatorWorker' => 'PhabricatorWorker',
@@ -3451,6 +3456,7 @@ phutil_register_library_map(array(
     'DrydockResourceViewController' => 'DrydockResourceController',
     'DrydockSFTPFilesystemInterface' => 'DrydockFilesystemInterface',
     'DrydockSSHCommandInterface' => 'DrydockCommandInterface',
+    'DrydockSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'DrydockWebrootInterface' => 'DrydockInterface',
     'DrydockWorkingCopyBlueprintImplementation' => 'DrydockBlueprintImplementation',
     'FeedConduitAPIMethod' => 'ConduitAPIMethod',
@@ -4430,6 +4436,7 @@ phutil_register_library_map(array(
     'PhabricatorDoorkeeperApplication' => 'PhabricatorApplication',
     'PhabricatorDraft' => 'PhabricatorDraftDAO',
     'PhabricatorDraftDAO' => 'PhabricatorLiskDAO',
+    'PhabricatorDraftSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'PhabricatorDrydockApplication' => 'PhabricatorApplication',
     'PhabricatorEdgeConfig' => 'PhabricatorEdgeConstants',
     'PhabricatorEdgeCycleException' => 'Exception',
@@ -4486,6 +4493,7 @@ phutil_register_library_map(array(
     'PhabricatorFeedManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorFeedPublicStreamController' => 'PhabricatorFeedController',
     'PhabricatorFeedQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'PhabricatorFeedSchemaSpec' => 'PhabricatorConfigSchemaSpec',
     'PhabricatorFeedSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'PhabricatorFeedStory' => array(
       'PhabricatorPolicyInterface',

src/applications/auth/storage/PhabricatorAuthProviderConfig.php

@@ -31,6 +31,9 @@ final class PhabricatorAuthProviderConfig extends PhabricatorAuthDAO
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
         'isEnabled' => 'bool',
+        'providerClass' => 'text128',
+        'providerType' => 'text64',
+        'providerDomain' => 'text128',
         'shouldAllowLogin' => 'bool',
         'shouldAllowRegistration' => 'bool',
         'shouldAllowLink' => 'bool',

src/applications/config/controller/PhabricatorConfigDatabaseController.php

@@ -688,6 +688,9 @@ final class PhabricatorConfigDatabaseController
       case PhabricatorConfigStorageSchema::STATUS_OKAY:
         $icon = 'fa-check-circle green';
         break;
+      case PhabricatorConfigStorageSchema::STATUS_NOTE:
+        $icon = 'fa-info-circle blue';
+        break;
       case PhabricatorConfigStorageSchema::STATUS_WARN:
         $icon = 'fa-exclamation-circle yellow';
         break;
@@ -735,6 +738,10 @@ final class PhabricatorConfigDatabaseController
 
         $status = PhabricatorConfigStorageSchema::getIssueStatus($issue);
         switch ($status) {
+          case PhabricatorConfigStorageSchema::STATUS_NOTE:
+            $icon = PHUIStatusItemView::ICON_INFO;
+            $color = 'blue';
+            break;
           case PhabricatorConfigStorageSchema::STATUS_WARN:
             $icon = PHUIStatusItemView::ICON_WARNING;
             $color = 'yellow';

src/applications/config/schema/PhabricatorConfigSchemaSpec.php

@@ -74,6 +74,10 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
     $table = $this->newTable($table_name);
 
     foreach ($columns as $name => $type) {
+      if ($type === null) {
+        continue;
+      }
+
       $details = $this->getDetailsForDataType($type);
       list($column_type, $charset, $collation, $nullable) = $details;
 

src/applications/config/schema/PhabricatorConfigStorageSchema.php

@@ -9,10 +9,12 @@ abstract class PhabricatorConfigStorageSchema extends Phobject {
   const ISSUE_COLUMNTYPE = 'columntype';
   const ISSUE_NULLABLE = 'nullable';
   const ISSUE_KEYCOLUMNS = 'keycolumns';
+  const ISSUE_SUBNOTE = 'subnote';
   const ISSUE_SUBWARN = 'subwarn';
   const ISSUE_SUBFAIL = 'subfail';
 
   const STATUS_OKAY = 'okay';
+  const STATUS_NOTE = 'note';
   const STATUS_WARN = 'warn';
   const STATUS_FAIL = 'fail';
 
@@ -55,6 +57,9 @@ abstract class PhabricatorConfigStorageSchema extends Phobject {
 
     foreach ($this->getSubschemata() as $sub) {
       switch ($sub->getStatus()) {
+        case self::STATUS_NOTE:
+          $issues[self::ISSUE_SUBNOTE] = self::ISSUE_SUBNOTE;
+          break;
         case self::STATUS_WARN:
           $issues[self::ISSUE_SUBWARN] = self::ISSUE_SUBWARN;
           break;
@@ -104,6 +109,8 @@ abstract class PhabricatorConfigStorageSchema extends Phobject {
         return pht('Wrong Nullable Setting');
       case self::ISSUE_KEYCOLUMNS:
         return pht('Key on Wrong Columns');
+      case self::ISSUE_SUBNOTE:
+        return pht('Subschemata Have Notices');
       case self::ISSUE_SUBWARN:
         return pht('Subschemata Have Warnings');
       case self::ISSUE_SUBFAIL:
@@ -129,6 +136,8 @@ abstract class PhabricatorConfigStorageSchema extends Phobject {
         return pht('This schema has the wrong nullable setting.');
       case self::ISSUE_KEYCOLUMNS:
         return pht('This schema is on the wrong columns.');
+      case self::ISSUE_SUBNOTE:
+        return pht('Subschemata have setup notices.');
       case self::ISSUE_SUBWARN:
         return pht('Subschemata have setup warnings.');
       case self::ISSUE_SUBFAIL:
@@ -144,13 +153,15 @@ abstract class PhabricatorConfigStorageSchema extends Phobject {
       case self::ISSUE_SUBFAIL:
         return self::STATUS_FAIL;
       case self::ISSUE_SURPLUS:
-      case self::ISSUE_CHARSET:
-      case self::ISSUE_COLLATION:
       case self::ISSUE_COLUMNTYPE:
       case self::ISSUE_SUBWARN:
       case self::ISSUE_KEYCOLUMNS:
       case self::ISSUE_NULLABLE:
         return self::STATUS_WARN;
+      case self::ISSUE_SUBNOTE:
+      case self::ISSUE_CHARSET:
+      case self::ISSUE_COLLATION:
+        return self::STATUS_NOTE;
       default:
         throw new Exception(pht('Unknown schema issue "%s"!', $issue));
     }
@@ -159,8 +170,10 @@ abstract class PhabricatorConfigStorageSchema extends Phobject {
   public static function getStatusSeverity($status) {
     switch ($status) {
       case self::STATUS_FAIL:
-        return 2;
+        return 3;
       case self::STATUS_WARN:
+        return 2;
+      case self::STATUS_NOTE:
         return 1;
       case self::STATUS_OKAY:
         return 0;

src/applications/dashboard/storage/PhabricatorDashboard.php

@@ -39,7 +39,11 @@ final class PhabricatorDashboard extends PhabricatorDashboardDAO
     return array(
       self::CONFIG_AUX_PHID => true,
       self::CONFIG_SERIALIZATION => array(
-        'layoutConfig' => self::SERIALIZATION_JSON),
+        'layoutConfig' => self::SERIALIZATION_JSON,
+      ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'name' => 'text255',
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/dashboard/storage/PhabricatorDashboardInstall.php

@@ -14,6 +14,19 @@ final class PhabricatorDashboardInstall
   protected $applicationClass;
   protected $dashboardPHID;
 
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'applicationClass' => 'text64',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'objectPHID' => array(
+          'columns' => array('objectPHID', 'applicationClass'),
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
   public static function getDashboard(
     PhabricatorUser $viewer,
     $object_phid,

src/applications/dashboard/storage/PhabricatorDashboardPanel.php

@@ -43,6 +43,11 @@ final class PhabricatorDashboardPanel
       self::CONFIG_SERIALIZATION => array(
         'properties' => self::SERIALIZATION_JSON,
       ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'name' => 'text255',
+        'panelType' => 'text64',
+        'isArchived' => 'bool',
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/doorkeeper/storage/DoorkeeperExternalObject.php

@@ -19,6 +19,20 @@ final class DoorkeeperExternalObject extends DoorkeeperDAO
       self::CONFIG_SERIALIZATION => array(
         'properties' => self::SERIALIZATION_JSON,
       ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'objectKey' => 'bytes12',
+        'applicationType' => 'text32',
+        'applicationDomain' => 'text32',
+        'objectType' => 'text32',
+        'objectID' => 'text64',
+        'objectURI' => 'text128?',
+        'importerPHID' => 'phid?',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_object' => array(
+          'columns' => array('objectKey'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/doorkeeper/storage/DoorkeeperSchemaSpec.php

@@ -0,0 +1,12 @@
+<?php
+
+final class DoorkeeperSchemaSpec
+  extends PhabricatorConfigSchemaSpec {
+
+  public function buildSchemata() {
+    $this->buildLiskSchemata('DoorkeeperDAO');
+
+    $this->buildEdgeSchemata(new DoorkeeperExternalObject());
+  }
+
+}

src/applications/draft/storage/PhabricatorDraft.php

@@ -14,6 +14,15 @@ final class PhabricatorDraft extends PhabricatorDraftDAO {
       self::CONFIG_SERIALIZATION => array(
         'metadata' => self::SERIALIZATION_JSON,
       ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'draftKey' => 'text64',
+        'draft' => 'text',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'authorPHID' => array(
+          'columns' => array('authorPHID', 'draftKey'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/draft/storage/PhabricatorDraftSchemaSpec.php

@@ -0,0 +1,10 @@
+<?php
+
+final class PhabricatorDraftSchemaSpec
+  extends PhabricatorConfigSchemaSpec {
+
+  public function buildSchemata() {
+    $this->buildLiskSchemata('PhabricatorDraftDAO');
+  }
+
+}

src/applications/drydock/storage/DrydockBlueprint.php

@@ -36,7 +36,11 @@ final class DrydockBlueprint extends DrydockDAO
       self::CONFIG_AUX_PHID => true,
       self::CONFIG_SERIALIZATION => array(
         'details' => self::SERIALIZATION_JSON,
-      )
+      ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'className' => 'text255',
+        'blueprintName' => 'text255',
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/drydock/storage/DrydockLease.php

@@ -43,6 +43,20 @@ final class DrydockLease extends DrydockDAO
       self::CONFIG_SERIALIZATION => array(
         'attributes'    => self::SERIALIZATION_JSON,
       ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'status' => 'uint32',
+        'until' => 'epoch?',
+        'resourceType' => 'text128',
+        'taskID' => 'id?',
+        'ownerPHID' => 'phid?',
+        'resourceID' => 'id?',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_phid' => null,
+        'phid' => array(
+          'columns' => array('phid'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/drydock/storage/DrydockLog.php

@@ -14,6 +14,11 @@ final class DrydockLog extends DrydockDAO
   public function getConfiguration() {
     return array(
       self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'resourceID' => 'id?',
+        'leaseID' => 'id?',
+        'message' => 'text',
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/drydock/storage/DrydockResource.php

@@ -23,6 +23,18 @@ final class DrydockResource extends DrydockDAO
         'attributes'    => self::SERIALIZATION_JSON,
         'capabilities'  => self::SERIALIZATION_JSON,
       ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'name' => 'text255',
+        'ownerPHID' => 'phid?',
+        'status' => 'uint32',
+        'type' => 'text64',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_phid' => null,
+        'phid' => array(
+          'columns' => array('phid'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/drydock/storage/DrydockSchemaSpec.php

@@ -0,0 +1,14 @@
+<?php
+
+final class DrydockSchemaSpec
+  extends PhabricatorConfigSchemaSpec {
+
+  public function buildSchemata() {
+    $this->buildLiskSchemata('DrydockDAO');
+
+    $this->buildTransactionSchema(
+      new DrydockBlueprintTransaction());
+
+  }
+
+}

src/applications/feed/storage/PhabricatorFeedSchemaSpec.php

@@ -0,0 +1,10 @@
+<?php
+
+final class PhabricatorFeedSchemaSpec
+  extends PhabricatorConfigSchemaSpec {
+
+  public function buildSchemata() {
+    $this->buildLiskSchemata('PhabricatorFeedDAO');
+  }
+
+}

src/applications/feed/storage/PhabricatorFeedStoryData.php

@@ -15,6 +15,19 @@ final class PhabricatorFeedStoryData extends PhabricatorFeedDAO {
       self::CONFIG_SERIALIZATION  => array(
         'storyData'  => self::SERIALIZATION_JSON,
       ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'chronologicalKey' => 'uint64',
+        'storyType' => 'text64',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_phid' => null,
+        'phid' => array(
+          'columns' => array('phid'),
+        ),
+        'chronologicalKey' => array(
+          'columns' => array('chronologicalKey'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/feed/storage/PhabricatorFeedStoryReference.php

@@ -9,6 +9,16 @@ final class PhabricatorFeedStoryReference extends PhabricatorFeedDAO {
     return array(
       self::CONFIG_IDS          => self::IDS_MANUAL,
       self::CONFIG_TIMESTAMPS   => false,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'chronologicalKey' => 'uint64',
+        'id' => null,
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'PRIMARY' => null,
+        'objectPHID' => array(
+          'columns' => array('objectPHID', 'chronologicalKey'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/applications/notification/storage/PhabricatorFeedStoryNotification.php

@@ -11,6 +11,17 @@ final class PhabricatorFeedStoryNotification extends PhabricatorFeedDAO {
     return array(
       self::CONFIG_IDS          => self::IDS_MANUAL,
       self::CONFIG_TIMESTAMPS   => false,
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'chronologicalKey' => 'uint64',
+        'hasViewed' => 'bool',
+        'id' => null,
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'PRIMARY' => null,
+        'userPHID' => array(
+          'columns' => array('userPHID', 'chronologicalKey'),
+        ),
+      ),
     ) + parent::getConfiguration();
   }
 

src/infrastructure/storage/lisk/LiskDAO.php

@@ -1750,9 +1750,8 @@ abstract class LiskDAO {
     $map = array();
     foreach ($this->getAllLiskProperties() as $property) {
       // First, use types specified explicitly in the table configuration.
-      $type = idx($custom_map, $property);
+      if (array_key_exists($property, $custom_map)) {
-      if ($type) {
+        $map[$property] = $custom_map[$property];
-        $map[$property] = $type;
         continue;
       }
 
@@ -1786,7 +1785,7 @@ abstract class LiskDAO {
       }
 
       // We don't know the type of this column.
-      $map[$property] = null;
+      $map[$property] = '<unknown>';
     }
 
     return $map;