From a115810912e7e42422543941b2e4a84c4d61b67c Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 17 Jul 2014 16:35:54 -0700 Subject: [PATCH] Give projects a proper on-demand datasource Summary: Fixes T5614. Ref T4420. Other than the "users" datasource and a couple of others, many datasources ignore what the user typed and just return all results, then rely on the client to filter them. This works fine for rarely used ("legalpad documents") or always small ("task priorities", "applications") datasets, but is something we should graudally move away from as datasets get larger. Add a token table to projects, populate it, and use it to drive the datasource query. Additionally, expose it on the applicationsearch UI. Test Plan: - Ran migration. - Manually checked the table. - Searched for projects by name from ApplicationSearch. - Searched for projects by name from typeahead. - Manually checked the typeahead response. Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T5614, T4420 Differential Revision: https://secure.phabricator.com/D9896 --- .../sql/autopatches/20140711.pnames.1.sql | 7 +++ .../sql/autopatches/20140711.pnames.2.php | 11 +++++ .../people/storage/PhabricatorUser.php | 20 ++------ .../PhabricatorProjectTransactionEditor.php | 5 ++ .../project/query/PhabricatorProjectQuery.php | 33 +++++++++++-- .../query/PhabricatorProjectSearchEngine.php | 13 +++++ .../project/storage/PhabricatorProject.php | 49 +++++++++++++++++++ .../PhabricatorProjectDatasource.php | 19 ++++++- .../PhabricatorTypeaheadDatasource.php | 11 +++++ 9 files changed, 145 insertions(+), 23 deletions(-) create mode 100644 resources/sql/autopatches/20140711.pnames.1.sql create mode 100644 resources/sql/autopatches/20140711.pnames.2.php diff --git a/resources/sql/autopatches/20140711.pnames.1.sql b/resources/sql/autopatches/20140711.pnames.1.sql new file mode 100644 index 0000000000..9fce73a47e --- /dev/null +++ b/resources/sql/autopatches/20140711.pnames.1.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_project.project_datasourcetoken ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + projectID INT UNSIGNED NOT NULL, + token VARCHAR(128) NOT NULL COLLATE utf8_general_ci, + UNIQUE KEY (token, projectID), + KEY (projectID) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/resources/sql/autopatches/20140711.pnames.2.php b/resources/sql/autopatches/20140711.pnames.2.php new file mode 100644 index 0000000000..ee3658384f --- /dev/null +++ b/resources/sql/autopatches/20140711.pnames.2.php @@ -0,0 +1,11 @@ +getName(); + echo "Updating project '{$name}'...\n"; + $project->updateDatasourceTokens(); +} + +echo "Done.\n"; diff --git a/src/applications/people/storage/PhabricatorUser.php b/src/applications/people/storage/PhabricatorUser.php index e12bef88f8..53a883106e 100644 --- a/src/applications/people/storage/PhabricatorUser.php +++ b/src/applications/people/storage/PhabricatorUser.php @@ -459,32 +459,18 @@ final class PhabricatorUser return $this; } - private static function tokenizeName($name) { - if (function_exists('mb_strtolower')) { - $name = mb_strtolower($name, 'UTF-8'); - } else { - $name = strtolower($name); - } - $name = trim($name); - if (!strlen($name)) { - return array(); - } - return preg_split('/\s+/', $name); - } - /** * Populate the nametoken table, which used to fetch typeahead results. When * a user types "linc", we want to match "Abraham Lincoln" from on-demand * typeahead sources. To do this, we need a separate table of name fragments. */ public function updateNameTokens() { - $tokens = array_merge( - self::tokenizeName($this->getRealName()), - self::tokenizeName($this->getUserName())); - $tokens = array_unique($tokens); $table = self::NAMETOKEN_TABLE; $conn_w = $this->establishConnection('w'); + $tokens = PhabricatorTypeaheadDatasource::tokenizeString( + $this->getUserName().' '.$this->getRealName()); + $sql = array(); foreach ($tokens as $token) { $sql[] = qsprintf( diff --git a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php index 386fb150d5..8fd66e77eb 100644 --- a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php +++ b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php @@ -125,6 +125,8 @@ final class PhabricatorProjectTransactionEditor ->setProjectPHID($object->getPHID()) ->save(); + $object->updateDatasourceTokens(); + // TODO -- delete all of the below once we sever automagical project // to phriction stuff if ($xaction->getOldValue() === null) { @@ -182,6 +184,9 @@ final class PhabricatorProjectTransactionEditor $rem_slug->delete(); } } + + $object->updateDatasourceTokens(); + return; case PhabricatorTransactions::TYPE_VIEW_POLICY: case PhabricatorTransactions::TYPE_EDIT_POLICY: diff --git a/src/applications/project/query/PhabricatorProjectQuery.php b/src/applications/project/query/PhabricatorProjectQuery.php index d4dc86e4ce..9f15c2c0ec 100644 --- a/src/applications/project/query/PhabricatorProjectQuery.php +++ b/src/applications/project/query/PhabricatorProjectQuery.php @@ -9,6 +9,7 @@ final class PhabricatorProjectQuery private $slugs; private $phrictionSlugs; private $names; + private $datasourceQuery; private $status = 'status-any'; const STATUS_ANY = 'status-any'; @@ -57,6 +58,11 @@ final class PhabricatorProjectQuery return $this; } + public function withDatasourceQuery($string) { + $this->datasourceQuery = $string; + return $this; + } + public function needMembers($need_members) { $this->needMembers = $need_members; return $this; @@ -286,7 +292,7 @@ final class PhabricatorProjectQuery } private function buildGroupClause($conn_r) { - if ($this->memberPHIDs) { + if ($this->memberPHIDs || $this->datasourceQuery) { return 'GROUP BY p.id'; } else { return $this->buildApplicationSearchGroupClause($conn_r); @@ -296,7 +302,7 @@ final class PhabricatorProjectQuery private function buildJoinClause($conn_r) { $joins = array(); - if (!$this->needMembers) { + if (!$this->needMembers !== null) { $joins[] = qsprintf( $conn_r, 'LEFT JOIN %T vm ON vm.src = p.phid AND vm.type = %d AND vm.dst = %s', @@ -305,7 +311,7 @@ final class PhabricatorProjectQuery $this->getViewer()->getPHID()); } - if ($this->memberPHIDs) { + if ($this->memberPHIDs !== null) { $joins[] = qsprintf( $conn_r, 'JOIN %T e ON e.src = p.phid AND e.type = %d', @@ -313,13 +319,32 @@ final class PhabricatorProjectQuery PhabricatorEdgeConfig::TYPE_PROJ_MEMBER); } - if ($this->slugs) { + if ($this->slugs !== null) { $joins[] = qsprintf( $conn_r, 'JOIN %T slug on slug.projectPHID = p.phid', id(new PhabricatorProjectSlug())->getTableName()); } + if ($this->datasourceQuery !== null) { + $tokens = PhabricatorTypeaheadDatasource::tokenizeString( + $this->datasourceQuery); + if (!$tokens) { + throw new PhabricatorEmptyQueryException(); + } + + $likes = array(); + foreach ($tokens as $token) { + $likes[] = qsprintf($conn_r, 'token.token LIKE %>', $token); + } + + $joins[] = qsprintf( + $conn_r, + 'JOIN %T token ON token.projectID = p.id AND (%Q)', + PhabricatorProject::TABLE_DATASOURCE_TOKEN, + '('.implode(') OR (', $likes).')'); + } + $joins[] = $this->buildApplicationSearchJoinClause($conn_r); return implode(' ', $joins); diff --git a/src/applications/project/query/PhabricatorProjectSearchEngine.php b/src/applications/project/query/PhabricatorProjectSearchEngine.php index 0122e17d0b..7120682cfb 100644 --- a/src/applications/project/query/PhabricatorProjectSearchEngine.php +++ b/src/applications/project/query/PhabricatorProjectSearchEngine.php @@ -21,7 +21,9 @@ final class PhabricatorProjectSearchEngine $saved->setParameter( 'memberPHIDs', $this->readUsersFromRequest($request, 'members')); + $saved->setParameter('status', $request->getStr('status')); + $saved->setParameter('name', $request->getStr('name')); $this->readCustomFieldsFromRequest($request, $saved); @@ -43,6 +45,11 @@ final class PhabricatorProjectSearchEngine $query->withStatus($status); } + $name = $saved->getParameter('name'); + if (strlen($name)) { + $query->withDatasourceQuery($name); + } + $this->applyCustomFieldsToQuery($query, $saved); return $query; @@ -59,8 +66,14 @@ final class PhabricatorProjectSearchEngine ->execute(); $status = $saved->getParameter('status'); + $name = $saved->getParameter('name'); $form + ->appendChild( + id(new AphrontFormTextControl()) + ->setName('name') + ->setLabel(pht('Name')) + ->setValue($name)) ->appendChild( id(new AphrontFormTokenizerControl()) ->setDatasource(new PhabricatorPeopleDatasource()) diff --git a/src/applications/project/storage/PhabricatorProject.php b/src/applications/project/storage/PhabricatorProject.php index 720c542392..fbcf331509 100644 --- a/src/applications/project/storage/PhabricatorProject.php +++ b/src/applications/project/storage/PhabricatorProject.php @@ -32,6 +32,8 @@ final class PhabricatorProject extends PhabricatorProjectDAO const DEFAULT_ICON = 'fa-briefcase'; const DEFAULT_COLOR = 'blue'; + const TABLE_DATASOURCE_TOKEN = 'project_datasourcetoken'; + public static function initializeNewProject(PhabricatorUser $actor) { return id(new PhabricatorProject()) ->setName('') @@ -219,6 +221,53 @@ final class PhabricatorProject extends PhabricatorProjectDAO return $this->color; } + public function save() { + $this->openTransaction(); + $result = parent::save(); + $this->updateDatasourceTokens(); + $this->saveTransaction(); + + return $result; + } + + public function updateDatasourceTokens() { + $table = self::TABLE_DATASOURCE_TOKEN; + $conn_w = $this->establishConnection('w'); + $id = $this->getID(); + + $slugs = queryfx_all( + $conn_w, + 'SELECT * FROM %T WHERE projectPHID = %s', + id(new PhabricatorProjectSlug())->getTableName(), + $this->getPHID()); + + $all_strings = ipull($slugs, 'slug'); + $all_strings[] = $this->getName(); + $all_strings = implode(' ', $all_strings); + + $tokens = PhabricatorTypeaheadDatasource::tokenizeString($all_strings); + + $sql = array(); + foreach ($tokens as $token) { + $sql[] = qsprintf($conn_w, '(%d, %s)', $id, $token); + } + + $this->openTransaction(); + queryfx( + $conn_w, + 'DELETE FROM %T WHERE projectID = %d', + $table, + $id); + + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { + queryfx( + $conn_w, + 'INSERT INTO %T (projectID, token) VALUES %Q', + $table, + $chunk); + } + $this->saveTransaction(); + } /* -( PhabricatorSubscribableInterface )----------------------------------- */ diff --git a/src/applications/project/typeahead/PhabricatorProjectDatasource.php b/src/applications/project/typeahead/PhabricatorProjectDatasource.php index f9bba5c33f..40ffc28724 100644 --- a/src/applications/project/typeahead/PhabricatorProjectDatasource.php +++ b/src/applications/project/typeahead/PhabricatorProjectDatasource.php @@ -13,22 +13,37 @@ final class PhabricatorProjectDatasource public function loadResults() { $viewer = $this->getViewer(); + $raw_query = $this->getRawQuery(); - $results = array(); + // Allow users to type "#qa" or "qa" to find "Quality Assurance". + $raw_query = ltrim($raw_query, '#'); + + if (!strlen($raw_query)) { + return array(); + } $projs = id(new PhabricatorProjectQuery()) ->setViewer($viewer) ->needImages(true) + ->needSlugs(true) + ->withDatasourceQuery($raw_query) ->execute(); + + $results = array(); foreach ($projs as $proj) { $closed = null; if ($proj->isArchived()) { $closed = pht('Archived'); } + $all_strings = mpull($proj->getSlugs(), 'getSlug'); + $all_strings[] = $proj->getName(); + $all_strings = implode(' ', $all_strings); + $proj_result = id(new PhabricatorTypeaheadResult()) - ->setName($proj->getName()) + ->setName($all_strings) + ->setDisplayName($proj->getName()) ->setDisplayType('Project') ->setURI('/tag/'.$proj->getPrimarySlug().'/') ->setPHID($proj->getPHID()) diff --git a/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php b/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php index 7ec636dda0..7ce5f5ab33 100644 --- a/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php +++ b/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php @@ -51,4 +51,15 @@ abstract class PhabricatorTypeaheadDatasource extends Phobject { abstract public function getDatasourceApplicationClass(); abstract public function loadResults(); + public static function tokenizeString($string) { + $string = phutil_utf8_strtolower($string); + $string = trim($string); + if (!strlen($string)) { + return array(); + } + + $tokens = preg_split('/\s+/', $string); + return array_unique($tokens); + } + }