From 2160c4561990f8938016e0ff85623ccbde03056f Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 14 Dec 2015 04:58:34 -0800 Subject: [PATCH] Implement an "Attachments" behavior for Conduit Search APIs Summary: Ref T9964. We have various kinds of secondary data on objects (like subscribers, projects, paste content, Owners paths, file attachments, etc) which is somewhat slow, or somewhat large, or both. Some approaches to handling this in the API include: - Always return all of it (very easy, but slow). - Require users to make separate API calls to get each piece of data (very simple, but inefficient and really cumbersome to use). - Implement a hierarchical query language like GraphQL (powerful, but very complex). - Kind of mix-and-match a half-power query language and some extra calls? (fairly simple, not too terrible?) We currently mix-and-match internally, with `->needStuff(true)`. This is not a general-purpose, full-power graph query language like GraphQL, and it occasionally does limit us. For example, there is no way to do this sort of thing: $conpherence_thread_query = id(new ConpherenceThreadQuery()) ->setViewer($viewer) // ... ->setNeedMessages(true) ->setWhenYouLoadTheMessagesTheyNeedProfilePictures(true); However, we almost never actually need to do this and when we do want to do it we usually don't //really// want to do it, so I don't think this is a major limit to the practical power of the system for the kinds of things we really want to do with it. Put another way, we have a lot of 1-level hierarchical queries (get pictures or repositories or projects or files or content for these objects) but few-to-no 2+ level queries (get files for these objects, then get all the projects for those files). So even though 1-level hierarchies are not a beautiful, general-purpose, fully-abstract system, they've worked well so far in practice and I'm comfortable moving forward with them in the API. If we do need N-level queries in the future, there is no technical reason we can't put GraphQL (or something similar) on top of this eventually, and this would represent a solid step toward that. However, I suspect we'll never need them. Upshot: I'm pretty happy with "->needX()" for all practical purposes, so this is just adding a way to say "->needX()" to the API. Specifically, you say: ``` { "attachments": { "subscribers": true, } } ``` ...and get back subscriber data. In the future (or for certain attachments), `true` might become a dictionary of extra parameters, if necessary, and could do so without breaking the API. Test Plan: - Ran queries to get attachments. {F1025449} Reviewers: chad Reviewed By: chad Maniphest Tasks: T9964 Differential Revision: https://secure.phabricator.com/D14772 --- src/__phutil_library_map__.php | 4 + .../PhabricatorApplicationSearchEngine.php | 85 ++++++++++++++---- .../PhabricatorSearchEngineAPIMethod.php | 90 +++++++++++++++++++ .../PhabricatorSearchEngineAttachment.php | 50 +++++++++++ .../PhabricatorSearchEngineExtension.php | 4 + ...torSubscriptionsSearchEngineAttachment.php | 84 +++++++++++++++++ ...atorSubscriptionsSearchEngineExtension.php | 6 ++ .../editfield/PhabricatorEditField.php | 4 +- 8 files changed, 308 insertions(+), 19 deletions(-) create mode 100644 src/applications/search/engineextension/PhabricatorSearchEngineAttachment.php create mode 100644 src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index e6eaec9be0..b59440ebc0 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3004,6 +3004,7 @@ phutil_register_library_map(array( 'PhabricatorSearchEditController' => 'applications/search/controller/PhabricatorSearchEditController.php', 'PhabricatorSearchEngine' => 'applications/search/engine/PhabricatorSearchEngine.php', 'PhabricatorSearchEngineAPIMethod' => 'applications/search/engine/PhabricatorSearchEngineAPIMethod.php', + 'PhabricatorSearchEngineAttachment' => 'applications/search/engineextension/PhabricatorSearchEngineAttachment.php', 'PhabricatorSearchEngineExtension' => 'applications/search/engineextension/PhabricatorSearchEngineExtension.php', 'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php', 'PhabricatorSearchEngineTestCase' => 'applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php', @@ -3149,6 +3150,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsListController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsListController.php', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSelfHeraldAction.php', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSubscribersHeraldAction.php', + 'PhabricatorSubscriptionsSearchEngineAttachment' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php', 'PhabricatorSubscriptionsSearchEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineExtension.php', 'PhabricatorSubscriptionsSubscribeEmailCommand' => 'applications/subscriptions/command/PhabricatorSubscriptionsSubscribeEmailCommand.php', 'PhabricatorSubscriptionsSubscribersPolicyRule' => 'applications/subscriptions/policyrule/PhabricatorSubscriptionsSubscribersPolicyRule.php', @@ -7321,6 +7323,7 @@ phutil_register_library_map(array( 'PhabricatorSearchEditController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchEngine' => 'Phobject', 'PhabricatorSearchEngineAPIMethod' => 'ConduitAPIMethod', + 'PhabricatorSearchEngineAttachment' => 'Phobject', 'PhabricatorSearchEngineExtension' => 'Phobject', 'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule', 'PhabricatorSearchEngineTestCase' => 'PhabricatorTestCase', @@ -7482,6 +7485,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsListController' => 'PhabricatorController', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', + 'PhabricatorSubscriptionsSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', 'PhabricatorSubscriptionsSearchEngineExtension' => 'PhabricatorSearchEngineExtension', 'PhabricatorSubscriptionsSubscribeEmailCommand' => 'MetaMTAEmailTransactionCommand', 'PhabricatorSubscriptionsSubscribersPolicyRule' => 'PhabricatorPolicyRule', diff --git a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php index 67caaa1035..99d8acef20 100644 --- a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php +++ b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php @@ -1123,10 +1123,25 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { $this->saveQuery($saved_query); - $query = $this->buildQueryFromSavedQuery($saved_query); $pager = $this->newPagerForSavedQuery($saved_query); + $attachments = $this->getConduitSearchAttachments(); + + // TODO: Validate this better. + $attachment_specs = $request->getValue('attachments'); + $attachments = array_select_keys( + $attachments, + array_keys($attachment_specs)); + + foreach ($attachments as $key => $attachment) { + $attachment->setViewer($viewer); + } + + foreach ($attachments as $key => $attachment) { + $attachment->willLoadAttachmentData($query, $attachment_specs[$key]); + } + $this->setQueryOrderForConduit($query, $request); $this->setPagerLimitForConduit($pager, $request); $this->setPagerOffsetsForConduit($pager, $request); @@ -1137,10 +1152,36 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { if ($objects) { $field_extensions = $this->getConduitFieldExtensions(); + $attachment_data = array(); + foreach ($attachments as $key => $attachment) { + $attachment_data[$key] = $attachment->loadAttachmentData( + $objects, + $attachment_specs[$key]); + } + foreach ($objects as $object) { - $data[] = $this->getObjectWireFormatForConduit( + $field_map = $this->getObjectWireFieldsForConduit( $object, $field_extensions); + + $attachment_map = array(); + foreach ($attachments as $key => $attachment) { + $attachment_map[$key] = $attachment->getAttachmentForObject( + $object, + $attachment_data[$key], + $attachment_specs[$key]); + } + + $id = (int)$object->getID(); + $phid = $object->getPHID(); + + $data[] = array( + 'id' => $id, + 'type' => phid_get_type($phid), + 'phid' => $phid, + 'fields' => $field_map, + 'attachments' => $attachment_map, + ); } } @@ -1264,21 +1305,6 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { } } - protected function getObjectWireFormatForConduit( - $object, - array $field_extensions) { - $phid = $object->getPHID(); - - return array( - 'id' => (int)$object->getID(), - 'type' => phid_get_type($phid), - 'phid' => $phid, - 'fields' => $this->getObjectWireFieldsForConduit( - $object, - $field_extensions), - ); - } - protected function getObjectWireFieldsForConduit( $object, array $field_extensions) { @@ -1291,4 +1317,29 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { return $fields; } + public function getConduitSearchAttachments() { + $extensions = $this->getEngineExtensions(); + + $attachments = array(); + foreach ($extensions as $extension) { + $extension_attachments = $extension->getSearchAttachments(); + foreach ($extension_attachments as $attachment) { + $attachment_key = $attachment->getAttachmentKey(); + if (isset($attachments[$attachment_key])) { + $other = $attachments[$attachment_key]; + throw new Exception( + pht( + 'Two search engine attachments (of classes "%s" and "%s") '. + 'specify the same attachment key ("%s"); keys must be unique.', + get_class($attachment), + get_class($other), + $attachment_key)); + } + $attachments[$attachment_key] = $attachment; + } + } + + return $attachments; + } + } diff --git a/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php b/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php index 051061d34c..78ba47229b 100644 --- a/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php +++ b/src/applications/search/engine/PhabricatorSearchEngineAPIMethod.php @@ -23,6 +23,7 @@ abstract class PhabricatorSearchEngineAPIMethod return array( 'queryKey' => 'optional string', 'constraints' => 'optional map', + 'attachments' => 'optional map', 'order' => 'optional order', ) + $this->getPagerParamTypes(); } @@ -58,6 +59,7 @@ abstract class PhabricatorSearchEngineAPIMethod $out[] = $this->buildConstraintsBox($engine); $out[] = $this->buildOrderBox($engine, $query); $out[] = $this->buildFieldsBox($engine); + $out[] = $this->buildAttachmentsBox($engine); $out[] = $this->buildPagingBox($engine); return $out; @@ -401,6 +403,94 @@ EOTEXT ->appendChild($table); } + private function buildAttachmentsBox( + PhabricatorApplicationSearchEngine $engine) { + + $info = pht(<<getConduitSearchAttachments(); + + $rows = array(); + foreach ($attachments as $key => $attachment) { + $rows[] = array( + $key, + $attachment->getAttachmentName(), + $attachment->getAttachmentDescription(), + ); + } + + $table = id(new AphrontTableView($rows)) + ->setHeaders( + array( + pht('Key'), + pht('Name'), + pht('Description'), + )) + ->setColumnClasses( + array( + 'prewrap', + 'pri', + 'wide', + )); + + return id(new PHUIObjectBoxView()) + ->setHeaderText(pht('Attachments')) + ->setCollapsed(true) + ->appendChild($this->buildRemarkup($info)) + ->appendChild($table); + } + private function buildPagingBox( PhabricatorApplicationSearchEngine $engine) { diff --git a/src/applications/search/engineextension/PhabricatorSearchEngineAttachment.php b/src/applications/search/engineextension/PhabricatorSearchEngineAttachment.php new file mode 100644 index 0000000000..3028250faf --- /dev/null +++ b/src/applications/search/engineextension/PhabricatorSearchEngineAttachment.php @@ -0,0 +1,50 @@ +viewer = $viewer; + return $this; + } + + final public function getViewer() { + return $this->viewer; + } + + final public function setSearchEngine( + PhabricatorApplicationSearchEngine $engine) { + $this->searchEngine = $engine; + return $this; + } + + final public function getSearchEngine() { + return $this->searchEngine; + } + + public function setAttachmentKey($attachment_key) { + $this->attachmentKey = $attachment_key; + return $this; + } + + public function getAttachmentKey() { + return $this->attachmentKey; + } + + abstract public function getAttachmentName(); + abstract public function getAttachmentDescription(); + + public function willLoadAttachmentData($query, $spec) { + return; + } + + public function loadAttachmentData(array $objects, $spec) { + return null; + } + + abstract public function getAttachmentForObject($object, $data, $spec); + +} diff --git a/src/applications/search/engineextension/PhabricatorSearchEngineExtension.php b/src/applications/search/engineextension/PhabricatorSearchEngineExtension.php index 1deba6063a..a5923734ea 100644 --- a/src/applications/search/engineextension/PhabricatorSearchEngineExtension.php +++ b/src/applications/search/engineextension/PhabricatorSearchEngineExtension.php @@ -40,6 +40,10 @@ abstract class PhabricatorSearchEngineExtension extends Phobject { return array(); } + public function getSearchAttachments() { + return array(); + } + public function applyConstraintsToQuery( $object, $query, diff --git a/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php new file mode 100644 index 0000000000..61c9c22ac0 --- /dev/null +++ b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php @@ -0,0 +1,84 @@ +withSourcePHIDs($object_phids) + ->withEdgeTypes(array($edge_type)); + $subscribers_query->execute(); + + $viewer = $this->getViewer(); + $viewer_phid = $viewer->getPHID(); + if ($viewer) { + $edges = id(new PhabricatorEdgeQuery()) + ->withSourcePHIDs($object_phids) + ->withEdgeTypes(array($edge_type)) + ->withDestinationPHIDs(array($viewer_phid)) + ->execute(); + + $viewer_map = array(); + foreach ($edges as $object_phid => $types) { + if ($types[$edge_type]) { + $viewer_map[$object_phid] = true; + } + } + } else { + $viewer_map = array(); + } + + return array( + 'subscribers.query' => $subscribers_query, + 'viewer.map' => $viewer_map, + ); + } + + public function getAttachmentForObject($object, $data, $spec) { + $subscribers_query = idx($data, 'subscribers.query'); + $viewer_map = idx($data, 'viewer.map'); + $object_phid = $object->getPHID(); + + $subscribed_phids = $subscribers_query->getDestinationPHIDs( + array($object_phid), + array(PhabricatorObjectHasSubscriberEdgeType::EDGECONST)); + $subscribed_count = count($subscribed_phids); + if ($subscribed_count > 10) { + $subscribed_phids = array_slice($subscribed_phids, 0, 10); + } + + $subscribed_phids = array_values($subscribed_phids); + + $viewer = $this->getViewer(); + $viewer_phid = $viewer->getPHID(); + + if (!$viewer_phid) { + $self_subscribed = false; + } else if (isset($viewer_map[$object_phid])) { + $self_subscribed = true; + } else if ($object->isAutomaticallySubscribed($viewer_phid)) { + $self_subscribed = true; + } else { + $self_subscribed = false; + } + + return array( + 'subscriberPHIDs' => $subscribed_phids, + 'subscriberCount' => $subscribed_count, + 'viewerIsSubscribed' => $self_subscribed, + ); + } + +} diff --git a/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineExtension.php b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineExtension.php index dc5838da6d..a2dc73d82a 100644 --- a/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineExtension.php +++ b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineExtension.php @@ -50,5 +50,11 @@ final class PhabricatorSubscriptionsSearchEngineExtension return $fields; } + public function getSearchAttachments() { + return array( + id(new PhabricatorSubscriptionsSearchEngineAttachment()) + ->setAttachmentKey('subscribers'), + ); + } } diff --git a/src/applications/transactions/editfield/PhabricatorEditField.php b/src/applications/transactions/editfield/PhabricatorEditField.php index 3cc2189ccb..4796c51467 100644 --- a/src/applications/transactions/editfield/PhabricatorEditField.php +++ b/src/applications/transactions/editfield/PhabricatorEditField.php @@ -516,7 +516,7 @@ abstract class PhabricatorEditField extends Phobject { $edit_type = $this->getEditType(); if ($edit_type === null) { - return null; + return array(); } return array($edit_type); @@ -526,7 +526,7 @@ abstract class PhabricatorEditField extends Phobject { $edit_type = $this->getEditType(); if ($edit_type === null) { - return null; + return array(); } return array($edit_type);