Import raw GitHub event data into Nuance

Summary:
Ref T10537. Ref T10538. This polls the GitHub events API and creates Nuance items from the raw data.

It does nothing useful with them.

Test Plan:
  - Polled GitHub.
  - Saw some items get created.
  - X-Poll-Interval seemed to work.
  - ETag seemed to work.
  - Recognizing when we hit items we've already seen seemed to work.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10537, T10538

Differential Revision: https://secure.phabricator.com/D15440

resources/sql/autopatches/20160308.nuance.06.label.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_item
+  DROP sourceLabel;

resources/sql/autopatches/20160308.nuance.07.itemtype.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_item
+  ADD itemType VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20160308.nuance.08.itemkey.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_item
+  ADD itemKey VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20160308.nuance.09.itemcontainer.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_item
+  ADD itemContainerKey VARCHAR(64) COLLATE {$COLLATE_TEXT};

resources/sql/autopatches/20160308.nuance.10.itemkeyu.sql

@@ -0,0 +1,2 @@
+UPDATE {$NAMESPACE}_nuance.nuance_item
+  SET itemKey = id WHERE itemKey = '';

resources/sql/autopatches/20160308.nuance.11.requestor.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_item
+  CHANGE requestorPHID requestorPHID VARBINARY(64);

resources/sql/autopatches/20160308.nuance.12.queue.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_item
+  CHANGE queuePHID queuePHID VARBINARY(64);

src/__phutil_library_map__.php

@@ -1419,7 +1419,6 @@ phutil_register_library_map(array(
     'NuanceConduitAPIMethod' => 'applications/nuance/conduit/NuanceConduitAPIMethod.php',
     'NuanceConsoleController' => 'applications/nuance/controller/NuanceConsoleController.php',
     'NuanceController' => 'applications/nuance/controller/NuanceController.php',
-    'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
     'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php',
     'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php',
     'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php',
@@ -1428,10 +1427,13 @@ phutil_register_library_map(array(
     'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php',
     'NuanceImportCursorPHIDType' => 'applications/nuance/phid/NuanceImportCursorPHIDType.php',
     'NuanceItem' => 'applications/nuance/storage/NuanceItem.php',
+    'NuanceItemController' => 'applications/nuance/controller/NuanceItemController.php',
     'NuanceItemEditController' => 'applications/nuance/controller/NuanceItemEditController.php',
     'NuanceItemEditor' => 'applications/nuance/editor/NuanceItemEditor.php',
+    'NuanceItemListController' => 'applications/nuance/controller/NuanceItemListController.php',
     'NuanceItemPHIDType' => 'applications/nuance/phid/NuanceItemPHIDType.php',
     'NuanceItemQuery' => 'applications/nuance/query/NuanceItemQuery.php',
+    'NuanceItemSearchEngine' => 'applications/nuance/query/NuanceItemSearchEngine.php',
     'NuanceItemTransaction' => 'applications/nuance/storage/NuanceItemTransaction.php',
     'NuanceItemTransactionComment' => 'applications/nuance/storage/NuanceItemTransactionComment.php',
     'NuanceItemTransactionQuery' => 'applications/nuance/query/NuanceItemTransactionQuery.php',
@@ -5668,12 +5670,14 @@ phutil_register_library_map(array(
     'NuanceConduitAPIMethod' => 'ConduitAPIMethod',
     'NuanceConsoleController' => 'NuanceController',
     'NuanceController' => 'PhabricatorController',
-    'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
     'NuanceDAO' => 'PhabricatorLiskDAO',
     'NuanceGitHubRepositoryImportCursor' => 'NuanceImportCursor',
     'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition',
     'NuanceImportCursor' => 'Phobject',
-    'NuanceImportCursorData' => 'NuanceDAO',
+    'NuanceImportCursorData' => array(
+      'NuanceDAO',
+      'PhabricatorPolicyInterface',
+    ),
     'NuanceImportCursorDataQuery' => 'NuanceQuery',
     'NuanceImportCursorPHIDType' => 'PhabricatorPHIDType',
     'NuanceItem' => array(
@@ -5681,10 +5685,13 @@ phutil_register_library_map(array(
       'PhabricatorPolicyInterface',
       'PhabricatorApplicationTransactionInterface',
     ),
+    'NuanceItemController' => 'NuanceController',
     'NuanceItemEditController' => 'NuanceController',
     'NuanceItemEditor' => 'PhabricatorApplicationTransactionEditor',
+    'NuanceItemListController' => 'NuanceItemController',
     'NuanceItemPHIDType' => 'PhabricatorPHIDType',
     'NuanceItemQuery' => 'NuanceQuery',
+    'NuanceItemSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'NuanceItemTransaction' => 'NuanceTransaction',
     'NuanceItemTransactionComment' => 'PhabricatorApplicationTransactionComment',
     'NuanceItemTransactionQuery' => 'PhabricatorApplicationTransactionQuery',

src/applications/nuance/application/PhabricatorNuanceApplication.php

@@ -40,6 +40,7 @@ final class PhabricatorNuanceApplication extends PhabricatorApplication {
       '/nuance/' => array(
         '' => 'NuanceConsoleController',
         'item/' => array(
+          $this->getQueryRoutePattern() => 'NuanceItemListController',
           'view/(?P<id>[1-9]\d*)/' => 'NuanceItemViewController',
           'edit/(?P<id>[1-9]\d*)/' => 'NuanceItemEditController',
           'new/'                   => 'NuanceItemEditController',

src/applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php

@@ -1,73 +0,0 @@
-<?php
-
-final class NuanceCreateItemConduitAPIMethod extends NuanceConduitAPIMethod {
-
-  public function getAPIMethodName() {
-    return 'nuance.createitem';
-  }
-
-  public function getMethodDescription() {
-    return pht('Create a new item.');
-  }
-
-  protected function defineParamTypes() {
-    return array(
-      'requestorPHID' => 'required string',
-      'sourcePHID'    => 'required string',
-      'ownerPHID'     => 'optional string',
-    );
-  }
-
-  protected function defineReturnType() {
-    return 'nonempty dict';
-  }
-
-  protected function defineErrorTypes() {
-    return array(
-      'ERR-NO-REQUESTOR-PHID' => pht('Items must have a requestor.'),
-      'ERR-NO-SOURCE-PHID' => pht('Items must have a source.'),
-    );
-  }
-
-  protected function execute(ConduitAPIRequest $request) {
-    $source_phid = $request->getValue('sourcePHID');
-    $owner_phid = $request->getValue('ownerPHID');
-    $requestor_phid = $request->getValue('requestorPHID');
-
-    $user = $request->getUser();
-
-    $item = NuanceItem::initializeNewItem();
-    $xactions = array();
-
-    if ($source_phid) {
-      $xactions[] = id(new NuanceItemTransaction())
-        ->setTransactionType(NuanceItemTransaction::TYPE_SOURCE)
-        ->setNewValue($source_phid);
-    } else {
-      throw new ConduitException('ERR-NO-SOURCE-PHID');
-    }
-
-    if ($owner_phid) {
-      $xactions[] = id(new NuanceItemTransaction())
-        ->setTransactionType(NuanceItemTransaction::TYPE_OWNER)
-        ->setNewValue($owner_phid);
-    }
-
-    if ($requestor_phid) {
-      $xactions[] = id(new NuanceItemTransaction())
-        ->setTransactionType(NuanceItemTransaction::TYPE_REQUESTOR)
-        ->setNewValue($requestor_phid);
-    } else {
-      throw new ConduitException('ERR-NO-REQUESTOR-PHID');
-    }
-
-    $source = PhabricatorContentSource::newFromConduitRequest($request);
-    $editor = id(new NuanceItemEditor())
-      ->setActor($user)
-      ->setContentSource($source)
-      ->applyTransactions($item, $xactions);
-
-    return $item->toDictionary();
-  }
-
-}

src/applications/nuance/controller/NuanceConsoleController.php

@@ -26,6 +26,13 @@ final class NuanceConsoleController extends NuanceController {
         ->setIcon('fa-filter')
         ->addAttribute(pht('Manage Nuance sources.')));
 
+    $menu->addItem(
+      id(new PHUIObjectItemView())
+        ->setHeader(pht('Items'))
+        ->setHref($this->getApplicationURI('item/'))
+        ->setIcon('fa-clone')
+        ->addAttribute(pht('Manage Nuance items.')));
+
     $crumbs = $this->buildApplicationCrumbs();
     $crumbs->addTextCrumb(pht('Console'));
 

src/applications/nuance/controller/NuanceItemController.php

@@ -0,0 +1,11 @@
+<?php
+
+abstract class NuanceItemController
+  extends NuanceController {
+
+  public function buildApplicationMenu() {
+    return $this->newApplicationMenu()
+      ->setSearchEngine(new NuanceItemSearchEngine());
+  }
+
+}

src/applications/nuance/controller/NuanceItemEditController.php

@@ -74,7 +74,7 @@ final class NuanceItemEditController extends NuanceController {
       $viewer->renderHandle($item->getQueuePHID()));
 
     $source = $item->getSource();
-    $definition = $source->requireDefinition();
+    $definition = $source->getDefinition();
 
     $definition->renderItemEditProperties(
       $viewer,

src/applications/nuance/controller/NuanceItemListController.php

@@ -0,0 +1,12 @@
+<?php
+
+final class NuanceItemListController
+  extends NuanceItemController {
+
+  public function handleRequest(AphrontRequest $request) {
+    return id(new NuanceItemSearchEngine())
+      ->setController($this)
+      ->buildResponse();
+  }
+
+}

src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php

@@ -37,22 +37,35 @@ final class NuanceGitHubRepositoryImportCursor
   }
 
   protected function pullDataFromSource() {
+    $viewer = $this->getViewer();
+    $now = PhabricatorTime::getNow();
+
     $source = $this->getSource();
 
     $user = $source->getSourceProperty('github.user');
     $repository = $source->getSourceProperty('github.repository');
     $api_token = $source->getSourceProperty('github.token');
 
+    // This API only supports fetching 10 pages of 30 events each, for a total
+    // of 300 events.
+    $etag = null;
+    $new_items = array();
+    $hit_known_items = false;
+    for ($page = 1; $page <= 10; $page++) {
       $uri = "/repos/{$user}/{$repository}/events";
-    $data = array();
+      $data = array(
+        'page' => $page,
+      );
 
       $future = id(new PhutilGitHubFuture())
         ->setAccessToken($api_token)
         ->setRawGitHubQuery($uri, $data);
 
-    $etag = $this->getCursorProperty('github.poll.etag');
-    if ($etag) {
-      $future->addHeader('If-None-Match', $etag);
+      if ($page == 1) {
+        $cursor_etag = $this->getCursorProperty('github.poll.etag');
+        if ($cursor_etag) {
+          $future->addHeader('If-None-Match', $cursor_etag);
+        }
       }
 
       $this->logInfo(
@@ -65,16 +78,105 @@ final class NuanceGitHubRepositoryImportCursor
       // body isn't valid.
       $this->updateRateLimits($response);
 
-    // This means we hit a rate limit or a "Not Modified" because of the "ETag"
-    // header. In either case, we should bail out.
+      if ($response->getStatus()->getStatusCode() == 304) {
+        $this->logInfo(
+          pht(
+            'Received a 304 Not Modified from GitHub, no new events.'));
+      }
+
+      // This means we hit a rate limit or a "Not Modified" because of the
+      // "ETag" header. In either case, we should bail out.
       if ($response->getStatus()->isError()) {
-      // TODO: Save cursor data!
+        $this->updatePolling($response, $now, false);
+        $this->getCursorData()->save();
         return false;
       }
 
-    $this->updateETag($response);
+      if ($page == 1) {
+        $etag = $response->getHeaderValue('ETag');
+      }
 
-    var_dump($response->getBody());
+      $records = $response->getBody();
+      foreach ($records as $record) {
+        $item = $this->newNuanceItemFromGitHubEvent($record);
+        $item_key = $item->getItemKey();
+
+        $this->logInfo(
+          pht(
+            'Fetched event "%s".',
+            $item_key));
+
+        $new_items[$item->getItemKey()] = $item;
+      }
+
+      if ($new_items) {
+        $existing = id(new NuanceItemQuery())
+          ->setViewer($viewer)
+          ->withSourcePHIDs(array($source->getPHID()))
+          ->withItemKeys(array_keys($new_items))
+          ->execute();
+        $existing = mpull($existing, null, 'getItemKey');
+        foreach ($new_items as $key => $new_item) {
+          if (isset($existing[$key])) {
+            unset($new_items[$key]);
+            $hit_known_items = true;
+
+            $this->logInfo(
+              pht(
+                'Event "%s" is previously known.',
+                $key));
+          }
+        }
+      }
+
+      if ($hit_known_items) {
+        break;
+      }
+
+      if (count($records) < 30) {
+        break;
+      }
+    }
+
+    // TODO: When we go through the whole queue without hitting anything we
+    // have seen before, we should record some sort of global event so we
+    // can tell the user when the bridging started or was interrupted?
+    if (!$hit_known_items) {
+      $already_polled = $this->getCursorProperty('github.polled');
+      if ($already_polled) {
+        // TODO: This is bad: we missed some items, maybe because too much
+        // stuff happened too fast or the daemons were broken for a long
+        // time.
+      } else {
+        // TODO: This is OK, we're doing the initial import.
+      }
+    }
+
+    if ($etag !== null) {
+      $this->updateETag($etag);
+    }
+
+    $this->updatePolling($response, $now, true);
+
+    $source->openTransaction();
+      foreach ($new_items as $new_item) {
+        $new_item->save();
+      }
+      $this->getCursorData()->save();
+    $source->saveTransaction();
+
+    foreach ($new_items as $new_item) {
+      PhabricatorWorker::scheduleTask(
+        'NuanceImportWorker',
+        array(
+          'itemPHID' => $new_item->getPHID(),
+        ),
+        array(
+          'objectPHID' => $new_item->getPHID(),
+        ));
+    }
+
+    return false;
   }
 
   private function updateRateLimits(PhutilGitHubResponse $response) {
@@ -100,8 +202,7 @@ final class NuanceGitHubRepositoryImportCursor
         new PhutilNumber($limit_reset - $now)));
   }
 
-  private function updateETag(PhutilGitHubResponse $response) {
-    $etag = $response->getHeaderValue('ETag');
+  private function updateETag($etag) {
 
     $this->setCursorProperty('github.poll.etag', $etag);
 
@@ -111,4 +212,54 @@ final class NuanceGitHubRepositoryImportCursor
         $etag));
   }
 
+  private function updatePolling(
+    PhutilGitHubResponse $response,
+    $start,
+    $success) {
+
+    if ($success) {
+      $this->setCursorProperty('github.polled', true);
+    }
+
+    $poll_interval = (int)$response->getHeaderValue('X-Poll-Interval');
+    $poll_ttl = $start + $poll_interval;
+    $this->setCursorProperty('github.poll.ttl', $poll_ttl);
+
+    $now = PhabricatorTime::getNow();
+
+    $this->logInfo(
+      pht(
+        'Set API poll TTL to +%s second(s) (%s second(s) from now).',
+        new PhutilNumber($poll_interval),
+        new PhutilNumber($poll_ttl - $now)));
+  }
+
+  private function newNuanceItemFromGitHubEvent(array $record) {
+    $source = $this->getSource();
+
+    $id = $record['id'];
+    $item_key = "github.event.{$id}";
+
+    $container_key = null;
+
+    $issue_id = idxv(
+      $record,
+      array(
+        'payload',
+        'issue',
+        'id',
+      ));
+    if ($issue_id) {
+      $container_key = "github.issue.{$issue_id}";
+    }
+
+    return NuanceItem::initializeNewItem()
+      ->setStatus(NuanceItem::STATUS_IMPORTING)
+      ->setSourcePHID($source->getPHID())
+      ->setItemType('github.event')
+      ->setItemKey($item_key)
+      ->setItemContainerKey($container_key)
+      ->setItemProperty('api.raw', $record);
+  }
+
 }

src/applications/nuance/cursor/NuanceImportCursor.php

@@ -5,6 +5,7 @@ abstract class NuanceImportCursor extends Phobject {
   private $cursorData;
   private $cursorKey;
   private $source;
+  private $viewer;
 
   abstract protected function shouldPullDataFromSource();
   abstract protected function pullDataFromSource();
@@ -40,6 +41,15 @@ abstract class NuanceImportCursor extends Phobject {
     return $this->cursorKey;
   }
 
+  public function setViewer($viewer) {
+    $this->viewer = $viewer;
+    return $this;
+  }
+
+  public function getViewer() {
+    return $this->viewer;
+  }
+
   final public function importFromSource() {
     if (!$this->shouldPullDataFromSource()) {
       return false;

src/applications/nuance/query/NuanceItemQuery.php

@@ -6,6 +6,9 @@ final class NuanceItemQuery
   private $ids;
   private $phids;
   private $sourcePHIDs;
+  private $itemTypes;
+  private $itemKeys;
+  private $containerKeys;
 
   public function withIDs(array $ids) {
     $this->ids = $ids;
@@ -22,6 +25,21 @@ final class NuanceItemQuery
     return $this;
   }
 
+  public function withItemTypes(array $item_types) {
+    $this->itemTypes = $item_types;
+    return $this;
+  }
+
+  public function withItemKeys(array $item_keys) {
+    $this->itemKeys = $item_keys;
+    return $this;
+  }
+
+  public function withItemContainerKeys(array $container_keys) {
+    $this->containerKeys = $container_keys;
+    return $this;
+  }
+
   public function newResultObject() {
     return new NuanceItem();
   }
@@ -79,6 +97,27 @@ final class NuanceItemQuery
         $this->phids);
     }
 
+    if ($this->itemTypes !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'itemType IN (%Ls)',
+        $this->itemTypes);
+    }
+
+    if ($this->itemKeys !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'itemKey IN (%Ls)',
+        $this->itemKeys);
+    }
+
+    if ($this->containerKeys !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'itemContainerKey IN (%Ls)',
+        $this->containerKeys);
+    }
+
     return $where;
   }
 

src/applications/nuance/query/NuanceItemSearchEngine.php

@@ -0,0 +1,81 @@
+<?php
+
+final class NuanceItemSearchEngine
+  extends PhabricatorApplicationSearchEngine {
+
+  public function getApplicationClassName() {
+    return 'PhabricatorNuanceApplication';
+  }
+
+  public function getResultTypeDescription() {
+    return pht('Nuance Items');
+  }
+
+  public function newQuery() {
+    return new NuanceItemQuery();
+  }
+
+  protected function buildQueryFromParameters(array $map) {
+    $query = $this->newQuery();
+
+    return $query;
+  }
+
+  protected function buildCustomSearchFields() {
+    return array(
+    );
+  }
+
+  protected function getURI($path) {
+    return '/nuance/item/'.$path;
+  }
+
+  protected function getBuiltinQueryNames() {
+    $names = array(
+      'all' => pht('All Items'),
+    );
+
+    return $names;
+  }
+
+  public function buildSavedQueryFromBuiltin($query_key) {
+    $query = $this->newSavedQuery();
+    $query->setQueryKey($query_key);
+
+    switch ($query_key) {
+      case 'all':
+        return $query;
+    }
+
+    return parent::buildSavedQueryFromBuiltin($query_key);
+  }
+
+  protected function renderResultList(
+    array $items,
+    PhabricatorSavedQuery $query,
+    array $handles) {
+    assert_instances_of($items, 'NuanceItem');
+
+    $viewer = $this->requireViewer();
+
+    $list = new PHUIObjectItemListView();
+    $list->setUser($viewer);
+    foreach ($items as $item) {
+      $view = id(new PHUIObjectItemView())
+        ->setObjectName(pht('Item %d', $item->getID()))
+        ->setHeader($item->getDisplayName())
+        ->setHref($item->getURI());
+
+      $view->addIcon('none', $item->getItemType());
+
+      $list->addItem($view);
+    }
+
+    $result = new PhabricatorApplicationSearchResultView();
+    $result->setObjectList($list);
+    $result->setNoDataString(pht('No items found.'));
+
+    return $result;
+  }
+
+}

src/applications/nuance/source/NuanceSourceDefinition.php

@@ -53,14 +53,15 @@ abstract class NuanceSourceDefinition extends Phobject {
         pht('This source has no input cursors.'));
     }
 
+    $viewer = PhabricatorUser::getOmnipotentUser();
     $source = $this->getSource();
     $cursors = $this->newImportCursors();
 
     $data = id(new NuanceImportCursorDataQuery())
-      ->setViewer(PhabricatorUser::getOmnipotentUser())
+      ->setViewer($viewer)
       ->withSourcePHIDs(array($source->getPHID()))
       ->execute();
-    $data = mpull($data, 'getCursorKey');
+    $data = mpull($data, null, 'getCursorKey');
 
     $map = array();
     foreach ($cursors as $cursor) {
@@ -102,14 +103,15 @@ abstract class NuanceSourceDefinition extends Phobject {
 
       $map[$key] = $cursor;
 
-      $cursor->setSource($source);
-
       $cursor_data = idx($data, $key);
       if (!$cursor_data) {
         $cursor_data = $cursor->newEmptyCursorData($source);
       }
 
-      $cursor->setCursorData($cursor_data);
+      $cursor
+        ->setViewer($viewer)
+        ->setSource($source)
+        ->setCursorData($cursor_data);
     }
 
     return $cursors;

src/applications/nuance/storage/NuanceImportCursorData.php

@@ -1,7 +1,8 @@
 <?php
 
 final class NuanceImportCursorData
-  extends NuanceDAO {
+  extends NuanceDAO
+  implements PhabricatorPolicyInterface {
 
   protected $sourcePHID;
   protected $cursorKey;
@@ -41,4 +42,29 @@ final class NuanceImportCursorData
     return $this;
   }
 
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    switch ($capability) {
+      case PhabricatorPolicyCapability::CAN_VIEW:
+        return PhabricatorPolicies::POLICY_USER;
+    }
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return false;
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return null;
+  }
+
 }

src/applications/nuance/storage/NuanceItem.php

@@ -6,18 +6,21 @@ final class NuanceItem
     PhabricatorPolicyInterface,
     PhabricatorApplicationTransactionInterface {
 
-  const STATUS_OPEN     = 0;
-  const STATUS_ASSIGNED = 10;
-  const STATUS_CLOSED   = 20;
+  const STATUS_IMPORTING = 'importing';
+  const STATUS_OPEN = 'open';
+  const STATUS_ASSIGNED = 'assigned';
+  const STATUS_CLOSED = 'closed';
 
   protected $status;
   protected $ownerPHID;
   protected $requestorPHID;
   protected $sourcePHID;
-  protected $sourceLabel;
+  protected $queuePHID;
+  protected $itemType;
+  protected $itemKey;
+  protected $itemContainerKey;
   protected $data = array();
   protected $mailKey;
-  protected $queuePHID;
 
   private $source = self::ATTACHABLE;
 
@@ -34,8 +37,12 @@ final class NuanceItem
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
         'ownerPHID' => 'phid?',
-        'sourceLabel' => 'text255?',
-        'status' => 'uint32',
+        'requestorPHID' => 'phid?',
+        'queuePHID' => 'phid?',
+        'itemType' => 'text64',
+        'itemKey' => 'text64',
+        'itemContainerKey' => 'text64?',
+        'status' => 'text32',
         'mailKey' => 'bytes20',
       ),
       self::CONFIG_KEY_SCHEMA => array(
@@ -51,6 +58,13 @@ final class NuanceItem
         'key_queue' => array(
           'columns' => array('queuePHID', 'status'),
         ),
+        'key_container' => array(
+          'columns' => array('sourcePHID', 'itemContainerKey'),
+        ),
+        'key_item' => array(
+          'columns' => array('sourcePHID', 'itemKey'),
+          'unique' => true,
+        ),
       ),
     ) + parent::getConfiguration();
   }
@@ -72,15 +86,7 @@ final class NuanceItem
   }
 
   public function getLabel(PhabricatorUser $viewer) {
-    // this is generated at the time the item is created based on
-    // the configuration from the item source. It is typically
-    // something like 'Twitter'.
-    $source_label = $this->getSourceLabel();
-
-    return pht(
-      'Item via %s @ %s.',
-      $source_label,
-      phabricator_datetime($this->getDateCreated(), $viewer));
+    return pht('TODO: An Item');
   }
 
   public function getRequestor() {
@@ -99,11 +105,11 @@ final class NuanceItem
     $this->source = $source;
   }
 
-  public function getNuanceProperty($key, $default = null) {
+  public function getItemProperty($key, $default = null) {
     return idx($this->data, $key, $default);
   }
 
-  public function setNuanceProperty($key, $value) {
+  public function setItemProperty($key, $value) {
     $this->data[$key] = $value;
     return $this;
   }
@@ -135,17 +141,8 @@ final class NuanceItem
     return null;
   }
 
-  public function toDictionary() {
-    return array(
-      'id' => $this->getID(),
-      'phid' => $this->getPHID(),
-      'ownerPHID' => $this->getOwnerPHID(),
-      'requestorPHID' => $this->getRequestorPHID(),
-      'sourcePHID' => $this->getSourcePHID(),
-      'sourceLabel' => $this->getSourceLabel(),
-      'dateCreated' => $this->getDateCreated(),
-      'dateModified' => $this->getDateModified(),
-    );
+  public function getDisplayName() {
+    return pht('An Item');
   }